aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm64')
-rw-r--r--arch/arm64/Kconfig119
-rw-r--r--arch/arm64/Kconfig.debug6
-rw-r--r--arch/arm64/Kconfig.platforms12
-rw-r--r--arch/arm64/Makefile7
-rw-r--r--arch/arm64/boot/dts/Makefile7
-rw-r--r--arch/arm64/boot/dts/altera/Makefile5
-rw-r--r--arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi358
-rw-r--r--arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts39
-rw-r--r--arch/arm64/boot/dts/amd/amd-overdrive.dts1
-rw-r--r--arch/arm64/boot/dts/apm/Makefile1
-rw-r--r--arch/arm64/boot/dts/apm/apm-merlin.dts72
-rw-r--r--arch/arm64/boot/dts/apm/apm-mustang.dts12
-rw-r--r--arch/arm64/boot/dts/apm/apm-shadowcat.dtsi271
-rw-r--r--arch/arm64/boot/dts/apm/apm-storm.dtsi55
-rw-r--r--arch/arm64/boot/dts/arm/juno-base.dtsi59
-rw-r--r--arch/arm64/boot/dts/arm/juno-motherboard.dtsi27
-rw-r--r--arch/arm64/boot/dts/arm/juno-r1.dts70
-rw-r--r--arch/arm64/boot/dts/arm/juno.dts50
-rw-r--r--arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts2
l---------arch/arm64/boot/dts/arm/vexpress-v2m-rs1.dtsi1
-rw-r--r--arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi103
-rw-r--r--arch/arm64/boot/dts/exynos/exynos7.dtsi7
-rw-r--r--arch/arm64/boot/dts/freescale/Makefile4
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts204
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts166
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls2080a-simu.dts (renamed from arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts)25
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi515
-rw-r--r--arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi163
-rw-r--r--arch/arm64/boot/dts/hisilicon/Makefile2
-rw-r--r--arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts7
-rw-r--r--arch/arm64/boot/dts/hisilicon/hi6220.dtsi43
-rw-r--r--arch/arm64/boot/dts/hisilicon/hip05-d02.dts36
-rw-r--r--arch/arm64/boot/dts/hisilicon/hip05.dtsi271
-rw-r--r--arch/arm64/boot/dts/hisilicon/hip05_hns.dtsi191
-rw-r--r--arch/arm64/boot/dts/marvell/Makefile1
-rw-r--r--arch/arm64/boot/dts/marvell/berlin4ct-stb.dts66
-rw-r--r--arch/arm64/boot/dts/marvell/berlin4ct.dtsi120
-rw-r--r--arch/arm64/boot/dts/mediatek/mt8173-evb.dts18
-rw-r--r--arch/arm64/boot/dts/mediatek/mt8173.dtsi62
-rw-r--r--arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi30
-rw-r--r--arch/arm64/boot/dts/qcom/msm8916-pins.dtsi76
-rw-r--r--arch/arm64/boot/dts/qcom/msm8916.dtsi47
-rw-r--r--arch/arm64/configs/defconfig24
-rw-r--r--arch/arm64/crypto/aes-ce-cipher.c2
-rw-r--r--arch/arm64/include/asm/acpi.h6
-rw-r--r--arch/arm64/include/asm/arch_gicv3.h170
-rw-r--r--arch/arm64/include/asm/assembler.h11
-rw-r--r--arch/arm64/include/asm/atomic.h65
-rw-r--r--arch/arm64/include/asm/atomic_ll_sc.h100
-rw-r--r--arch/arm64/include/asm/atomic_lse.h195
-rw-r--r--arch/arm64/include/asm/barrier.h16
-rw-r--r--arch/arm64/include/asm/cache.h2
-rw-r--r--arch/arm64/include/asm/cacheflush.h7
-rw-r--r--arch/arm64/include/asm/cachetype.h4
-rw-r--r--arch/arm64/include/asm/cmpxchg.h279
-rw-r--r--arch/arm64/include/asm/compat.h3
-rw-r--r--arch/arm64/include/asm/cpu.h4
-rw-r--r--arch/arm64/include/asm/cpufeature.h95
-rw-r--r--arch/arm64/include/asm/cputype.h30
-rw-r--r--arch/arm64/include/asm/dcc.h55
-rw-r--r--arch/arm64/include/asm/dma-mapping.h28
-rw-r--r--arch/arm64/include/asm/fixmap.h7
-rw-r--r--arch/arm64/include/asm/hw_breakpoint.h9
-rw-r--r--arch/arm64/include/asm/hwcap.h8
-rw-r--r--arch/arm64/include/asm/irq.h14
-rw-r--r--arch/arm64/include/asm/kasan.h38
-rw-r--r--arch/arm64/include/asm/kernel-pgtable.h83
-rw-r--r--arch/arm64/include/asm/kvm_arm.h16
-rw-r--r--arch/arm64/include/asm/kvm_emulate.h8
-rw-r--r--arch/arm64/include/asm/kvm_host.h5
-rw-r--r--arch/arm64/include/asm/memory.h7
-rw-r--r--arch/arm64/include/asm/mmu.h15
-rw-r--r--arch/arm64/include/asm/mmu_context.h113
-rw-r--r--arch/arm64/include/asm/page.h27
-rw-r--r--arch/arm64/include/asm/pgalloc.h1
-rw-r--r--arch/arm64/include/asm/pgtable-hwdef.h48
-rw-r--r--arch/arm64/include/asm/pgtable.h34
-rw-r--r--arch/arm64/include/asm/pmu.h83
-rw-r--r--arch/arm64/include/asm/processor.h2
-rw-r--r--arch/arm64/include/asm/ptrace.h16
-rw-r--r--arch/arm64/include/asm/string.h16
-rw-r--r--arch/arm64/include/asm/sysreg.h157
-rw-r--r--arch/arm64/include/asm/thread_info.h5
-rw-r--r--arch/arm64/include/asm/tlb.h26
-rw-r--r--arch/arm64/include/asm/tlbflush.h18
-rw-r--r--arch/arm64/include/uapi/asm/kvm.h2
-rw-r--r--arch/arm64/kernel/Makefile11
-rw-r--r--arch/arm64/kernel/acpi.c44
-rw-r--r--arch/arm64/kernel/arm64ksyms.c3
-rw-r--r--arch/arm64/kernel/armv8_deprecated.c16
-rw-r--r--arch/arm64/kernel/asm-offsets.c2
-rw-r--r--arch/arm64/kernel/cpu_errata.c20
-rw-r--r--arch/arm64/kernel/cpufeature.c866
-rw-r--r--arch/arm64/kernel/cpuinfo.c261
-rw-r--r--arch/arm64/kernel/debug-monitors.c10
-rw-r--r--arch/arm64/kernel/efi-entry.S10
-rw-r--r--arch/arm64/kernel/efi-stub.c68
-rw-r--r--arch/arm64/kernel/efi.c38
-rw-r--r--arch/arm64/kernel/entry.S2
-rw-r--r--arch/arm64/kernel/fpsimd.c16
-rw-r--r--arch/arm64/kernel/head.S78
-rw-r--r--arch/arm64/kernel/hw_breakpoint.c19
-rw-r--r--arch/arm64/kernel/image.h38
-rw-r--r--arch/arm64/kernel/irq.c62
-rw-r--r--arch/arm64/kernel/module.c16
-rw-r--r--arch/arm64/kernel/perf_event.c1066
-rw-r--r--arch/arm64/kernel/process.c3
-rw-r--r--arch/arm64/kernel/psci.c14
-rw-r--r--arch/arm64/kernel/setup.c245
-rw-r--r--arch/arm64/kernel/smp.c24
-rw-r--r--arch/arm64/kernel/stacktrace.c6
-rw-r--r--arch/arm64/kernel/suspend.c34
-rw-r--r--arch/arm64/kernel/time.c8
-rw-r--r--arch/arm64/kernel/traps.c15
-rw-r--r--arch/arm64/kernel/vdso/Makefile3
-rw-r--r--arch/arm64/kernel/vmlinux.lds.S6
-rw-r--r--arch/arm64/kvm/Kconfig9
-rw-r--r--arch/arm64/kvm/hyp.S22
-rw-r--r--arch/arm64/kvm/inject_fault.c2
-rw-r--r--arch/arm64/kvm/reset.c2
-rw-r--r--arch/arm64/kvm/sys_regs.c12
-rw-r--r--arch/arm64/lib/copy_from_user.S78
-rw-r--r--arch/arm64/lib/copy_in_user.S67
-rw-r--r--arch/arm64/lib/copy_template.S193
-rw-r--r--arch/arm64/lib/copy_to_user.S67
-rw-r--r--arch/arm64/lib/memchr.S2
-rw-r--r--arch/arm64/lib/memcmp.S2
-rw-r--r--arch/arm64/lib/memcpy.S184
-rw-r--r--arch/arm64/lib/memmove.S9
-rw-r--r--arch/arm64/lib/memset.S5
-rw-r--r--arch/arm64/lib/strcmp.S2
-rw-r--r--arch/arm64/lib/strlen.S2
-rw-r--r--arch/arm64/lib/strncmp.S2
-rw-r--r--arch/arm64/mm/Makefile3
-rw-r--r--arch/arm64/mm/cache.S10
-rw-r--r--arch/arm64/mm/context.c236
-rw-r--r--arch/arm64/mm/dma-mapping.c480
-rw-r--r--arch/arm64/mm/dump.c18
-rw-r--r--arch/arm64/mm/fault.c2
-rw-r--r--arch/arm64/mm/init.c19
-rw-r--r--arch/arm64/mm/kasan_init.c165
-rw-r--r--arch/arm64/mm/mmu.c165
-rw-r--r--arch/arm64/mm/pageattr.c2
-rw-r--r--arch/arm64/mm/pgd.c2
-rw-r--r--arch/arm64/mm/proc.S14
-rw-r--r--arch/arm64/net/bpf_jit.h3
-rw-r--r--arch/arm64/net/bpf_jit_comp.c104
147 files changed, 7210 insertions, 2862 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 07d1811aa03f..e55848c1edf4 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -27,6 +27,7 @@ config ARM64
27 select CPU_PM if (SUSPEND || CPU_IDLE) 27 select CPU_PM if (SUSPEND || CPU_IDLE)
28 select DCACHE_WORD_ACCESS 28 select DCACHE_WORD_ACCESS
29 select EDAC_SUPPORT 29 select EDAC_SUPPORT
30 select FRAME_POINTER
30 select GENERIC_ALLOCATOR 31 select GENERIC_ALLOCATOR
31 select GENERIC_CLOCKEVENTS 32 select GENERIC_CLOCKEVENTS
32 select GENERIC_CLOCKEVENTS_BROADCAST 33 select GENERIC_CLOCKEVENTS_BROADCAST
@@ -48,6 +49,7 @@ config ARM64
48 select HAVE_ARCH_AUDITSYSCALL 49 select HAVE_ARCH_AUDITSYSCALL
49 select HAVE_ARCH_BITREVERSE 50 select HAVE_ARCH_BITREVERSE
50 select HAVE_ARCH_JUMP_LABEL 51 select HAVE_ARCH_JUMP_LABEL
52 select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP
51 select HAVE_ARCH_KGDB 53 select HAVE_ARCH_KGDB
52 select HAVE_ARCH_SECCOMP_FILTER 54 select HAVE_ARCH_SECCOMP_FILTER
53 select HAVE_ARCH_TRACEHOOK 55 select HAVE_ARCH_TRACEHOOK
@@ -75,6 +77,7 @@ config ARM64
75 select HAVE_PERF_USER_STACK_DUMP 77 select HAVE_PERF_USER_STACK_DUMP
76 select HAVE_RCU_TABLE_FREE 78 select HAVE_RCU_TABLE_FREE
77 select HAVE_SYSCALL_TRACEPOINTS 79 select HAVE_SYSCALL_TRACEPOINTS
80 select IOMMU_DMA if IOMMU_SUPPORT
78 select IRQ_DOMAIN 81 select IRQ_DOMAIN
79 select IRQ_FORCED_THREADING 82 select IRQ_FORCED_THREADING
80 select MODULES_USE_ELF_RELA 83 select MODULES_USE_ELF_RELA
@@ -169,10 +172,12 @@ config FIX_EARLYCON_MEM
169 172
170config PGTABLE_LEVELS 173config PGTABLE_LEVELS
171 int 174 int
175 default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36
172 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 176 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42
173 default 3 if ARM64_64K_PAGES && ARM64_VA_BITS_48 177 default 3 if ARM64_64K_PAGES && ARM64_VA_BITS_48
174 default 3 if ARM64_4K_PAGES && ARM64_VA_BITS_39 178 default 3 if ARM64_4K_PAGES && ARM64_VA_BITS_39
175 default 4 if ARM64_4K_PAGES && ARM64_VA_BITS_48 179 default 3 if ARM64_16K_PAGES && ARM64_VA_BITS_47
180 default 4 if !ARM64_64K_PAGES && ARM64_VA_BITS_48
176 181
177source "init/Kconfig" 182source "init/Kconfig"
178 183
@@ -311,6 +316,27 @@ config ARM64_ERRATUM_832075
311 316
312 If unsure, say Y. 317 If unsure, say Y.
313 318
319config ARM64_ERRATUM_834220
320 bool "Cortex-A57: 834220: Stage 2 translation fault might be incorrectly reported in presence of a Stage 1 fault"
321 depends on KVM
322 default y
323 help
324 This option adds an alternative code sequence to work around ARM
325 erratum 834220 on Cortex-A57 parts up to r1p2.
326
327 Affected Cortex-A57 parts might report a Stage 2 translation
328 fault as the result of a Stage 1 fault for load crossing a
329 page boundary when there is a permission or device memory
330 alignment fault at Stage 1 and a translation fault at Stage 2.
331
332 The workaround is to verify that the Stage 1 translation
333 doesn't generate a fault before handling the Stage 2 fault.
334 Please note that this does not necessarily enable the workaround,
335 as it depends on the alternative framework, which will only patch
336 the kernel if an affected CPU is detected.
337
338 If unsure, say Y.
339
314config ARM64_ERRATUM_845719 340config ARM64_ERRATUM_845719
315 bool "Cortex-A53: 845719: a load might read incorrect data" 341 bool "Cortex-A53: 845719: a load might read incorrect data"
316 depends on COMPAT 342 depends on COMPAT
@@ -348,6 +374,33 @@ config ARM64_ERRATUM_843419
348 374
349 If unsure, say Y. 375 If unsure, say Y.
350 376
377config CAVIUM_ERRATUM_22375
378 bool "Cavium erratum 22375, 24313"
379 default y
380 help
381 Enable workaround for erratum 22375, 24313.
382
383 This implements two gicv3-its errata workarounds for ThunderX. Both
384 with small impact affecting only ITS table allocation.
385
386 erratum 22375: only alloc 8MB table size
387 erratum 24313: ignore memory access type
388
389 The fixes are in ITS initialization and basically ignore memory access
390 type and table size provided by the TYPER and BASER registers.
391
392 If unsure, say Y.
393
394config CAVIUM_ERRATUM_23154
395 bool "Cavium erratum 23154: Access to ICC_IAR1_EL1 is not sync'ed"
396 default y
397 help
398 The gicv3 of ThunderX requires a modified version for
399 reading the IAR status to ensure data synchronization
400 (access to icc_iar1_el1 is not sync'ed before and after).
401
402 If unsure, say Y.
403
351endmenu 404endmenu
352 405
353 406
@@ -362,25 +415,37 @@ config ARM64_4K_PAGES
362 help 415 help
363 This feature enables 4KB pages support. 416 This feature enables 4KB pages support.
364 417
418config ARM64_16K_PAGES
419 bool "16KB"
420 help
421 The system will use 16KB pages support. AArch32 emulation
422 requires applications compiled with 16K (or a multiple of 16K)
423 aligned segments.
424
365config ARM64_64K_PAGES 425config ARM64_64K_PAGES
366 bool "64KB" 426 bool "64KB"
367 help 427 help
368 This feature enables 64KB pages support (4KB by default) 428 This feature enables 64KB pages support (4KB by default)
369 allowing only two levels of page tables and faster TLB 429 allowing only two levels of page tables and faster TLB
370 look-up. AArch32 emulation is not available when this feature 430 look-up. AArch32 emulation requires applications compiled
371 is enabled. 431 with 64K aligned segments.
372 432
373endchoice 433endchoice
374 434
375choice 435choice
376 prompt "Virtual address space size" 436 prompt "Virtual address space size"
377 default ARM64_VA_BITS_39 if ARM64_4K_PAGES 437 default ARM64_VA_BITS_39 if ARM64_4K_PAGES
438 default ARM64_VA_BITS_47 if ARM64_16K_PAGES
378 default ARM64_VA_BITS_42 if ARM64_64K_PAGES 439 default ARM64_VA_BITS_42 if ARM64_64K_PAGES
379 help 440 help
380 Allows choosing one of multiple possible virtual address 441 Allows choosing one of multiple possible virtual address
381 space sizes. The level of translation table is determined by 442 space sizes. The level of translation table is determined by
382 a combination of page size and virtual address space size. 443 a combination of page size and virtual address space size.
383 444
445config ARM64_VA_BITS_36
446 bool "36-bit" if EXPERT
447 depends on ARM64_16K_PAGES
448
384config ARM64_VA_BITS_39 449config ARM64_VA_BITS_39
385 bool "39-bit" 450 bool "39-bit"
386 depends on ARM64_4K_PAGES 451 depends on ARM64_4K_PAGES
@@ -389,6 +454,10 @@ config ARM64_VA_BITS_42
389 bool "42-bit" 454 bool "42-bit"
390 depends on ARM64_64K_PAGES 455 depends on ARM64_64K_PAGES
391 456
457config ARM64_VA_BITS_47
458 bool "47-bit"
459 depends on ARM64_16K_PAGES
460
392config ARM64_VA_BITS_48 461config ARM64_VA_BITS_48
393 bool "48-bit" 462 bool "48-bit"
394 463
@@ -396,8 +465,10 @@ endchoice
396 465
397config ARM64_VA_BITS 466config ARM64_VA_BITS
398 int 467 int
468 default 36 if ARM64_VA_BITS_36
399 default 39 if ARM64_VA_BITS_39 469 default 39 if ARM64_VA_BITS_39
400 default 42 if ARM64_VA_BITS_42 470 default 42 if ARM64_VA_BITS_42
471 default 47 if ARM64_VA_BITS_47
401 default 48 if ARM64_VA_BITS_48 472 default 48 if ARM64_VA_BITS_48
402 473
403config CPU_BIG_ENDIAN 474config CPU_BIG_ENDIAN
@@ -427,15 +498,13 @@ config NR_CPUS
427 498
428config HOTPLUG_CPU 499config HOTPLUG_CPU
429 bool "Support for hot-pluggable CPUs" 500 bool "Support for hot-pluggable CPUs"
501 select GENERIC_IRQ_MIGRATION
430 help 502 help
431 Say Y here to experiment with turning CPUs off and on. CPUs 503 Say Y here to experiment with turning CPUs off and on. CPUs
432 can be controlled through /sys/devices/system/cpu. 504 can be controlled through /sys/devices/system/cpu.
433 505
434source kernel/Kconfig.preempt 506source kernel/Kconfig.preempt
435 507source kernel/Kconfig.hz
436config HZ
437 int
438 default 100
439 508
440config ARCH_HAS_HOLES_MEMORYMODEL 509config ARCH_HAS_HOLES_MEMORYMODEL
441 def_bool y if SPARSEMEM 510 def_bool y if SPARSEMEM
@@ -454,12 +523,8 @@ config HAVE_ARCH_PFN_VALID
454 def_bool ARCH_HAS_HOLES_MEMORYMODEL || !SPARSEMEM 523 def_bool ARCH_HAS_HOLES_MEMORYMODEL || !SPARSEMEM
455 524
456config HW_PERF_EVENTS 525config HW_PERF_EVENTS
457 bool "Enable hardware performance counter support for perf events" 526 def_bool y
458 depends on PERF_EVENTS 527 depends on ARM_PMU
459 default y
460 help
461 Enable hardware performance counter support for perf events. If
462 disabled, perf events will use software events only.
463 528
464config SYS_SUPPORTS_HUGETLBFS 529config SYS_SUPPORTS_HUGETLBFS
465 def_bool y 530 def_bool y
@@ -468,7 +533,7 @@ config ARCH_WANT_GENERAL_HUGETLB
468 def_bool y 533 def_bool y
469 534
470config ARCH_WANT_HUGE_PMD_SHARE 535config ARCH_WANT_HUGE_PMD_SHARE
471 def_bool y if !ARM64_64K_PAGES 536 def_bool y if ARM64_4K_PAGES || (ARM64_16K_PAGES && !ARM64_VA_BITS_36)
472 537
473config HAVE_ARCH_TRANSPARENT_HUGEPAGE 538config HAVE_ARCH_TRANSPARENT_HUGEPAGE
474 def_bool y 539 def_bool y
@@ -505,7 +570,25 @@ config XEN
505config FORCE_MAX_ZONEORDER 570config FORCE_MAX_ZONEORDER
506 int 571 int
507 default "14" if (ARM64_64K_PAGES && TRANSPARENT_HUGEPAGE) 572 default "14" if (ARM64_64K_PAGES && TRANSPARENT_HUGEPAGE)
573 default "12" if (ARM64_16K_PAGES && TRANSPARENT_HUGEPAGE)
508 default "11" 574 default "11"
575 help
576 The kernel memory allocator divides physically contiguous memory
577 blocks into "zones", where each zone is a power of two number of
578 pages. This option selects the largest power of two that the kernel
579 keeps in the memory allocator. If you need to allocate very large
580 blocks of physically contiguous memory, then you may need to
581 increase this value.
582
583 This config option is actually maximum order plus one. For example,
584 a value of 11 means that the largest free memory block is 2^10 pages.
585
586 We make sure that we can allocate upto a HugePage size for each configuration.
587 Hence we have :
588 MAX_ORDER = (PMD_SHIFT - PAGE_SHIFT) + 1 => PAGE_SHIFT - 2
589
590 However for 4K, we choose a higher default value, 11 as opposed to 10, giving us
591 4M allocations matching the default size used by generic code.
509 592
510menuconfig ARMV8_DEPRECATED 593menuconfig ARMV8_DEPRECATED
511 bool "Emulate deprecated/obsolete ARMv8 instructions" 594 bool "Emulate deprecated/obsolete ARMv8 instructions"
@@ -680,7 +763,7 @@ source "fs/Kconfig.binfmt"
680 763
681config COMPAT 764config COMPAT
682 bool "Kernel support for 32-bit EL0" 765 bool "Kernel support for 32-bit EL0"
683 depends on !ARM64_64K_PAGES || EXPERT 766 depends on ARM64_4K_PAGES || EXPERT
684 select COMPAT_BINFMT_ELF 767 select COMPAT_BINFMT_ELF
685 select HAVE_UID16 768 select HAVE_UID16
686 select OLD_SIGSUSPEND3 769 select OLD_SIGSUSPEND3
@@ -691,9 +774,9 @@ config COMPAT
691 the user helper functions, VFP support and the ptrace interface are 774 the user helper functions, VFP support and the ptrace interface are
692 handled appropriately by the kernel. 775 handled appropriately by the kernel.
693 776
694 If you also enabled CONFIG_ARM64_64K_PAGES, please be aware that you 777 If you use a page size other than 4KB (i.e, 16KB or 64KB), please be aware
695 will only be able to execute AArch32 binaries that were compiled with 778 that you will only be able to execute AArch32 binaries that were compiled
696 64k aligned segments. 779 with page size aligned segments.
697 780
698 If you want to execute 32-bit userspace applications, say Y. 781 If you want to execute 32-bit userspace applications, say Y.
699 782
diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug
index d6285ef9b5f9..04fb73b973f1 100644
--- a/arch/arm64/Kconfig.debug
+++ b/arch/arm64/Kconfig.debug
@@ -2,10 +2,6 @@ menu "Kernel hacking"
2 2
3source "lib/Kconfig.debug" 3source "lib/Kconfig.debug"
4 4
5config FRAME_POINTER
6 bool
7 default y
8
9config ARM64_PTDUMP 5config ARM64_PTDUMP
10 bool "Export kernel pagetable layout to userspace via debugfs" 6 bool "Export kernel pagetable layout to userspace via debugfs"
11 depends on DEBUG_KERNEL 7 depends on DEBUG_KERNEL
@@ -77,7 +73,7 @@ config DEBUG_RODATA
77 If in doubt, say Y 73 If in doubt, say Y
78 74
79config DEBUG_ALIGN_RODATA 75config DEBUG_ALIGN_RODATA
80 depends on DEBUG_RODATA && !ARM64_64K_PAGES 76 depends on DEBUG_RODATA && ARM64_4K_PAGES
81 bool "Align linker sections up to SECTION_SIZE" 77 bool "Align linker sections up to SECTION_SIZE"
82 help 78 help
83 If this option is enabled, sections that may potentially be marked as 79 If this option is enabled, sections that may potentially be marked as
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index 23800a19a7bc..4043c35962cc 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -7,6 +7,7 @@ config ARCH_BCM_IPROC
7 7
8config ARCH_BERLIN 8config ARCH_BERLIN
9 bool "Marvell Berlin SoC Family" 9 bool "Marvell Berlin SoC Family"
10 select ARCH_REQUIRE_GPIOLIB
10 select DW_APB_ICTL 11 select DW_APB_ICTL
11 help 12 help
12 This enables support for Marvell Berlin SoC Family 13 This enables support for Marvell Berlin SoC Family
@@ -28,10 +29,10 @@ config ARCH_EXYNOS7
28 help 29 help
29 This enables support for Samsung Exynos7 SoC family 30 This enables support for Samsung Exynos7 SoC family
30 31
31config ARCH_FSL_LS2085A 32config ARCH_LAYERSCAPE
32 bool "Freescale LS2085A SOC" 33 bool "ARMv8 based Freescale Layerscape SoC family"
33 help 34 help
34 This enables support for Freescale LS2085A SOC. 35 This enables support for the Freescale Layerscape SoC family.
35 36
36config ARCH_HISI 37config ARCH_HISI
37 bool "Hisilicon SoC Family" 38 bool "Hisilicon SoC Family"
@@ -66,6 +67,11 @@ config ARCH_SEATTLE
66 help 67 help
67 This enables support for AMD Seattle SOC Family 68 This enables support for AMD Seattle SOC Family
68 69
70config ARCH_STRATIX10
71 bool "Altera's Stratix 10 SoCFPGA Family"
72 help
73 This enables support for Altera's Stratix 10 SoCFPGA Family.
74
69config ARCH_TEGRA 75config ARCH_TEGRA
70 bool "NVIDIA Tegra SoC Family" 76 bool "NVIDIA Tegra SoC Family"
71 select ARCH_HAS_RESET_CONTROLLER 77 select ARCH_HAS_RESET_CONTROLLER
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index d10b5d483022..cd822d8454c0 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -55,6 +55,13 @@ else
55TEXT_OFFSET := 0x00080000 55TEXT_OFFSET := 0x00080000
56endif 56endif
57 57
58# KASAN_SHADOW_OFFSET = VA_START + (1 << (VA_BITS - 3)) - (1 << 61)
59# in 32-bit arithmetic
60KASAN_SHADOW_OFFSET := $(shell printf "0x%08x00000000\n" $$(( \
61 (0xffffffff & (-1 << ($(CONFIG_ARM64_VA_BITS) - 32))) \
62 + (1 << ($(CONFIG_ARM64_VA_BITS) - 32 - 3)) \
63 - (1 << (64 - 32 - 3)) )) )
64
58export TEXT_OFFSET GZFLAGS 65export TEXT_OFFSET GZFLAGS
59 66
60core-y += arch/arm64/kernel/ arch/arm64/mm/ 67core-y += arch/arm64/kernel/ arch/arm64/mm/
diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile
index d9f88330e7b0..eb3c42d97175 100644
--- a/arch/arm64/boot/dts/Makefile
+++ b/arch/arm64/boot/dts/Makefile
@@ -1,3 +1,4 @@
1dts-dirs += altera
1dts-dirs += amd 2dts-dirs += amd
2dts-dirs += apm 3dts-dirs += apm
3dts-dirs += arm 4dts-dirs += arm
@@ -14,3 +15,9 @@ dts-dirs += sprd
14dts-dirs += xilinx 15dts-dirs += xilinx
15 16
16subdir-y := $(dts-dirs) 17subdir-y := $(dts-dirs)
18
19dtstree := $(srctree)/$(src)
20
21dtb-$(CONFIG_OF_ALL_DTBS) := $(patsubst $(dtstree)/%.dts,%.dtb, $(foreach d,$(dts-dirs), $(wildcard $(dtstree)/$(d)/*.dts)))
22
23always := $(dtb-y)
diff --git a/arch/arm64/boot/dts/altera/Makefile b/arch/arm64/boot/dts/altera/Makefile
new file mode 100644
index 000000000000..d7a641698d77
--- /dev/null
+++ b/arch/arm64/boot/dts/altera/Makefile
@@ -0,0 +1,5 @@
1dtb-$(CONFIG_ARCH_STRATIX10) += socfpga_stratix10_socdk.dtb
2
3always := $(dtb-y)
4subdir-y := $(dts-dirs)
5clean-files := *.dtb
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
new file mode 100644
index 000000000000..445aa678f914
--- /dev/null
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
@@ -0,0 +1,358 @@
1/*
2 * Copyright Altera Corporation (C) 2015. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17/dts-v1/;
18
19/ {
20 compatible = "altr,socfpga-stratix10";
21 #address-cells = <2>;
22 #size-cells = <2>;
23
24 cpus {
25 #address-cells = <1>;
26 #size-cells = <0>;
27
28 cpu0: cpu@0 {
29 compatible = "arm,cortex-a53", "arm,armv8";
30 device_type = "cpu";
31 enable-method = "psci";
32 reg = <0x0>;
33 };
34
35 cpu1: cpu@1 {
36 compatible = "arm,cortex-a53", "arm,armv8";
37 device_type = "cpu";
38 enable-method = "psci";
39 reg = <0x1>;
40 };
41
42 cpu2: cpu@2 {
43 compatible = "arm,cortex-a53", "arm,armv8";
44 device_type = "cpu";
45 enable-method = "psci";
46 reg = <0x2>;
47 };
48
49 cpu3: cpu@3 {
50 compatible = "arm,cortex-a53", "arm,armv8";
51 device_type = "cpu";
52 enable-method = "psci";
53 reg = <0x3>;
54 };
55 };
56
57 pmu {
58 compatible = "arm,armv8-pmuv3";
59 interrupts = <0 120 8>,
60 <0 121 8>,
61 <0 122 8>,
62 <0 123 8>;
63 interrupt-affinity = <&cpu0>,
64 <&cpu1>,
65 <&cpu2>,
66 <&cpu3>;
67 };
68
69 psci {
70 compatible = "arm,psci-0.2";
71 method = "smc";
72 };
73
74 intc: intc@fffc1000 {
75 compatible = "arm,gic-400", "arm,cortex-a15-gic";
76 #interrupt-cells = <3>;
77 interrupt-controller;
78 reg = <0x0 0xfffc1000 0x1000>,
79 <0x0 0xfffc2000 0x2000>,
80 <0x0 0xfffc4000 0x2000>,
81 <0x0 0xfffc6000 0x2000>;
82 };
83
84 soc {
85 #address-cells = <1>;
86 #size-cells = <1>;
87 compatible = "simple-bus";
88 device_type = "soc";
89 interrupt-parent = <&intc>;
90 ranges = <0 0 0 0xffffffff>;
91
92 clkmgr@ffd1000 {
93 compatible = "altr,clk-mgr";
94 reg = <0xffd10000 0x1000>;
95 };
96
97 gmac0: ethernet@ff800000 {
98 compatible = "altr,socfpga-stmmac", "snps,dwmac-3.74a", "snps,dwmac";
99 reg = <0xff800000 0x2000>;
100 interrupts = <0 90 4>;
101 interrupt-names = "macirq";
102 mac-address = [00 00 00 00 00 00];
103 status = "disabled";
104 };
105
106 gmac1: ethernet@ff802000 {
107 compatible = "altr,socfpga-stmmac", "snps,dwmac-3.74a", "snps,dwmac";
108 reg = <0xff802000 0x2000>;
109 interrupts = <0 91 4>;
110 interrupt-names = "macirq";
111 mac-address = [00 00 00 00 00 00];
112 status = "disabled";
113 };
114
115 gmac2: ethernet@ff804000 {
116 compatible = "altr,socfpga-stmmac", "snps,dwmac-3.74a", "snps,dwmac";
117 reg = <0xff804000 0x2000>;
118 interrupts = <0 92 4>;
119 interrupt-names = "macirq";
120 mac-address = [00 00 00 00 00 00];
121 status = "disabled";
122 };
123
124 gpio0: gpio@ffc03200 {
125 #address-cells = <1>;
126 #size-cells = <0>;
127 compatible = "snps,dw-apb-gpio";
128 reg = <0xffc03200 0x100>;
129 status = "disabled";
130
131 porta: gpio-controller@0 {
132 compatible = "snps,dw-apb-gpio-port";
133 gpio-controller;
134 #gpio-cells = <2>;
135 snps,nr-gpios = <24>;
136 reg = <0>;
137 interrupt-controller;
138 #interrupt-cells = <2>;
139 interrupts = <0 110 4>;
140 };
141 };
142
143 gpio1: gpio@ffc03300 {
144 #address-cells = <1>;
145 #size-cells = <0>;
146 compatible = "snps,dw-apb-gpio";
147 reg = <0xffc03300 0x100>;
148 status = "disabled";
149
150 portb: gpio-controller@0 {
151 compatible = "snps,dw-apb-gpio-port";
152 gpio-controller;
153 #gpio-cells = <2>;
154 snps,nr-gpios = <24>;
155 reg = <0>;
156 interrupt-controller;
157 #interrupt-cells = <2>;
158 interrupts = <0 110 4>;
159 };
160 };
161
162 i2c0: i2c@ffc02800 {
163 #address-cells = <1>;
164 #size-cells = <0>;
165 compatible = "snps,designware-i2c";
166 reg = <0xffc02800 0x100>;
167 interrupts = <0 103 4>;
168 status = "disabled";
169 };
170
171 i2c1: i2c@ffc02900 {
172 #address-cells = <1>;
173 #size-cells = <0>;
174 compatible = "snps,designware-i2c";
175 reg = <0xffc02900 0x100>;
176 interrupts = <0 104 4>;
177 status = "disabled";
178 };
179
180 i2c2: i2c@ffc02a00 {
181 #address-cells = <1>;
182 #size-cells = <0>;
183 compatible = "snps,designware-i2c";
184 reg = <0xffc02a00 0x100>;
185 interrupts = <0 105 4>;
186 status = "disabled";
187 };
188
189 i2c3: i2c@ffc02b00 {
190 #address-cells = <1>;
191 #size-cells = <0>;
192 compatible = "snps,designware-i2c";
193 reg = <0xffc02b00 0x100>;
194 interrupts = <0 106 4>;
195 status = "disabled";
196 };
197
198 i2c4: i2c@ffc02c00 {
199 #address-cells = <1>;
200 #size-cells = <0>;
201 compatible = "snps,designware-i2c";
202 reg = <0xffc02c00 0x100>;
203 interrupts = <0 107 4>;
204 status = "disabled";
205 };
206
207 mmc: dwmmc0@ff808000 {
208 #address-cells = <1>;
209 #size-cells = <0>;
210 compatible = "altr,socfpga-dw-mshc";
211 reg = <0xff808000 0x1000>;
212 interrupts = <0 96 4>;
213 fifo-depth = <0x400>;
214 status = "disabled";
215 };
216
217 ocram: sram@ffe00000 {
218 compatible = "mmio-sram";
219 reg = <0xffe00000 0x100000>;
220 };
221
222 rst: rstmgr@ffd11000 {
223 #reset-cells = <1>;
224 compatible = "altr,rst-mgr";
225 reg = <0xffd11000 0x1000>;
226 };
227
228 spi0: spi@ffda4000 {
229 compatible = "snps,dw-apb-ssi";
230 #address-cells = <1>;
231 #size-cells = <0>;
232 reg = <0xffda4000 0x1000>;
233 interrupts = <0 101 4>;
234 num-chipselect = <4>;
235 bus-num = <0>;
236 status = "disabled";
237 };
238
239 spi1: spi@ffda5000 {
240 compatible = "snps,dw-apb-ssi";
241 #address-cells = <1>;
242 #size-cells = <0>;
243 reg = <0xffda5000 0x1000>;
244 interrupts = <0 102 4>;
245 num-chipselect = <4>;
246 bus-num = <0>;
247 status = "disabled";
248 };
249
250 sysmgr: sysmgr@ffd12000 {
251 compatible = "altr,sys-mgr", "syscon";
252 reg = <0xffd12000 0x1000>;
253 };
254
255 /* Local timer */
256 timer {
257 compatible = "arm,armv8-timer";
258 interrupts = <1 13 0xf01>,
259 <1 14 0xf01>,
260 <1 11 0xf01>,
261 <1 10 0xf01>;
262 };
263
264 timer0: timer0@ffc03000 {
265 compatible = "snps,dw-apb-timer";
266 interrupts = <0 113 4>;
267 reg = <0xffc03000 0x100>;
268 };
269
270 timer1: timer1@ffc03100 {
271 compatible = "snps,dw-apb-timer";
272 interrupts = <0 114 4>;
273 reg = <0xffc03100 0x100>;
274 };
275
276 timer2: timer2@ffd00000 {
277 compatible = "snps,dw-apb-timer";
278 interrupts = <0 115 4>;
279 reg = <0xffd00000 0x100>;
280 };
281
282 timer3: timer3@ffd00100 {
283 compatible = "snps,dw-apb-timer";
284 interrupts = <0 116 4>;
285 reg = <0xffd00100 0x100>;
286 };
287
288 uart0: serial0@ffc02000 {
289 compatible = "snps,dw-apb-uart";
290 reg = <0xffc02000 0x100>;
291 interrupts = <0 108 4>;
292 reg-shift = <2>;
293 reg-io-width = <4>;
294 status = "disabled";
295 };
296
297 uart1: serial1@ffc02100 {
298 compatible = "snps,dw-apb-uart";
299 reg = <0xffc02100 0x100>;
300 interrupts = <0 109 4>;
301 reg-shift = <2>;
302 reg-io-width = <4>;
303 status = "disabled";
304 };
305
306 usbphy0: usbphy@0 {
307 #phy-cells = <0>;
308 compatible = "usb-nop-xceiv";
309 status = "okay";
310 };
311
312 usb0: usb@ffb00000 {
313 compatible = "snps,dwc2";
314 reg = <0xffb00000 0x40000>;
315 interrupts = <0 93 4>;
316 phys = <&usbphy0>;
317 phy-names = "usb2-phy";
318 status = "disabled";
319 };
320
321 usb1: usb@ffb40000 {
322 compatible = "snps,dwc2";
323 reg = <0xffb40000 0x40000>;
324 interrupts = <0 94 4>;
325 phys = <&usbphy0>;
326 phy-names = "usb2-phy";
327 status = "disabled";
328 };
329
330 watchdog0: watchdog@ffd00200 {
331 compatible = "snps,dw-wdt";
332 reg = <0xffd00200 0x100>;
333 interrupts = <0 117 4>;
334 status = "disabled";
335 };
336
337 watchdog1: watchdog@ffd00300 {
338 compatible = "snps,dw-wdt";
339 reg = <0xffd00300 0x100>;
340 interrupts = <0 118 4>;
341 status = "disabled";
342 };
343
344 watchdog2: watchdog@ffd00400 {
345 compatible = "snps,dw-wdt";
346 reg = <0xffd00400 0x100>;
347 interrupts = <0 125 4>;
348 status = "disabled";
349 };
350
351 watchdog3: watchdog@ffd00500 {
352 compatible = "snps,dw-wdt";
353 reg = <0xffd00500 0x100>;
354 interrupts = <0 126 4>;
355 status = "disabled";
356 };
357 };
358};
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
new file mode 100644
index 000000000000..41ea2dba2fce
--- /dev/null
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
@@ -0,0 +1,39 @@
1/*
2 * Copyright Altera Corporation (C) 2015. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17/include/ "socfpga_stratix10.dtsi"
18
19/ {
20 model = "SoCFPGA Stratix 10 SoCDK";
21
22 aliases {
23 serial0 = &uart0;
24 };
25
26 chosen {
27 stdout-path = "serial0:115200n8";
28 };
29
30 memory {
31 device_type = "memory";
32 /* We expect the bootloader to fill in the reg */
33 reg = <0 0 0 0>;
34 };
35};
36
37&uart0 {
38 status = "okay";
39};
diff --git a/arch/arm64/boot/dts/amd/amd-overdrive.dts b/arch/arm64/boot/dts/amd/amd-overdrive.dts
index 564a3f7df71d..128fa942f09e 100644
--- a/arch/arm64/boot/dts/amd/amd-overdrive.dts
+++ b/arch/arm64/boot/dts/amd/amd-overdrive.dts
@@ -14,7 +14,6 @@
14 14
15 chosen { 15 chosen {
16 stdout-path = &serial0; 16 stdout-path = &serial0;
17 linux,pci-probe-only;
18 }; 17 };
19}; 18};
20 19
diff --git a/arch/arm64/boot/dts/apm/Makefile b/arch/arm64/boot/dts/apm/Makefile
index a2afabbc1717..c75f17a49471 100644
--- a/arch/arm64/boot/dts/apm/Makefile
+++ b/arch/arm64/boot/dts/apm/Makefile
@@ -1,4 +1,5 @@
1dtb-$(CONFIG_ARCH_XGENE) += apm-mustang.dtb 1dtb-$(CONFIG_ARCH_XGENE) += apm-mustang.dtb
2dtb-$(CONFIG_ARCH_XGENE) += apm-merlin.dtb
2 3
3always := $(dtb-y) 4always := $(dtb-y)
4subdir-y := $(dts-dirs) 5subdir-y := $(dts-dirs)
diff --git a/arch/arm64/boot/dts/apm/apm-merlin.dts b/arch/arm64/boot/dts/apm/apm-merlin.dts
new file mode 100644
index 000000000000..119a469bd189
--- /dev/null
+++ b/arch/arm64/boot/dts/apm/apm-merlin.dts
@@ -0,0 +1,72 @@
1/*
2 * dts file for AppliedMicro (APM) Merlin Board
3 *
4 * Copyright (C) 2015, Applied Micro Circuits Corporation
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation; either version 2 of
9 * the License, or (at your option) any later version.
10 */
11
12/dts-v1/;
13
14/include/ "apm-shadowcat.dtsi"
15
16/ {
17 model = "APM X-Gene Merlin board";
18 compatible = "apm,merlin", "apm,xgene-shadowcat";
19
20 chosen { };
21
22 memory {
23 device_type = "memory";
24 reg = < 0x1 0x00000000 0x0 0x80000000 >;
25 };
26
27 gpio-keys {
28 compatible = "gpio-keys";
29 button@1 {
30 label = "POWER";
31 linux,code = <116>;
32 linux,input-type = <0x1>;
33 interrupts = <0x0 0x28 0x1>;
34 };
35 };
36
37 poweroff_mbox: poweroff_mbox@10548000 {
38 compatible = "syscon";
39 reg = <0x0 0x10548000 0x0 0x30>;
40 };
41
42 poweroff: poweroff@10548010 {
43 compatible = "syscon-poweroff";
44 regmap = <&poweroff_mbox>;
45 offset = <0x10>;
46 mask = <0x1>;
47 };
48};
49
50&serial0 {
51 status = "ok";
52};
53
54&sata1 {
55 status = "ok";
56};
57
58&sata2 {
59 status = "ok";
60};
61
62&sata3 {
63 status = "ok";
64};
65
66&sgenet0 {
67 status = "ok";
68};
69
70&xgenet1 {
71 status = "ok";
72};
diff --git a/arch/arm64/boot/dts/apm/apm-mustang.dts b/arch/arm64/boot/dts/apm/apm-mustang.dts
index 4c55833d8a41..01cdeda93c3a 100644
--- a/arch/arm64/boot/dts/apm/apm-mustang.dts
+++ b/arch/arm64/boot/dts/apm/apm-mustang.dts
@@ -33,6 +33,18 @@
33 interrupts = <0x0 0x2d 0x1>; 33 interrupts = <0x0 0x2d 0x1>;
34 }; 34 };
35 }; 35 };
36
37 poweroff_mbox: poweroff_mbox@10548000 {
38 compatible = "syscon";
39 reg = <0x0 0x10548000 0x0 0x30>;
40 };
41
42 poweroff: poweroff@10548010 {
43 compatible = "syscon-poweroff";
44 regmap = <&poweroff_mbox>;
45 offset = <0x10>;
46 mask = <0x1>;
47 };
36}; 48};
37 49
38&pcie0clk { 50&pcie0clk {
diff --git a/arch/arm64/boot/dts/apm/apm-shadowcat.dtsi b/arch/arm64/boot/dts/apm/apm-shadowcat.dtsi
new file mode 100644
index 000000000000..c804f8f1f38c
--- /dev/null
+++ b/arch/arm64/boot/dts/apm/apm-shadowcat.dtsi
@@ -0,0 +1,271 @@
1/*
2 * dts file for AppliedMicro (APM) X-Gene Shadowcat SOC
3 *
4 * Copyright (C) 2015, Applied Micro Circuits Corporation
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation; either version 2 of
9 * the License, or (at your option) any later version.
10 */
11
12/ {
13 compatible = "apm,xgene-shadowcat";
14 interrupt-parent = <&gic>;
15 #address-cells = <2>;
16 #size-cells = <2>;
17
18 cpus {
19 #address-cells = <2>;
20 #size-cells = <0>;
21
22 cpu@000 {
23 device_type = "cpu";
24 compatible = "apm,strega", "arm,armv8";
25 reg = <0x0 0x000>;
26 enable-method = "spin-table";
27 cpu-release-addr = <0x1 0x0000fff8>;
28 };
29 cpu@001 {
30 device_type = "cpu";
31 compatible = "apm,strega", "arm,armv8";
32 reg = <0x0 0x001>;
33 enable-method = "spin-table";
34 cpu-release-addr = <0x1 0x0000fff8>;
35 };
36 cpu@100 {
37 device_type = "cpu";
38 compatible = "apm,strega", "arm,armv8";
39 reg = <0x0 0x100>;
40 enable-method = "spin-table";
41 cpu-release-addr = <0x1 0x0000fff8>;
42 };
43 cpu@101 {
44 device_type = "cpu";
45 compatible = "apm,strega", "arm,armv8";
46 reg = <0x0 0x101>;
47 enable-method = "spin-table";
48 cpu-release-addr = <0x1 0x0000fff8>;
49 };
50 cpu@200 {
51 device_type = "cpu";
52 compatible = "apm,strega", "arm,armv8";
53 reg = <0x0 0x200>;
54 enable-method = "spin-table";
55 cpu-release-addr = <0x1 0x0000fff8>;
56 };
57 cpu@201 {
58 device_type = "cpu";
59 compatible = "apm,strega", "arm,armv8";
60 reg = <0x0 0x201>;
61 enable-method = "spin-table";
62 cpu-release-addr = <0x1 0x0000fff8>;
63 };
64 cpu@300 {
65 device_type = "cpu";
66 compatible = "apm,strega", "arm,armv8";
67 reg = <0x0 0x300>;
68 enable-method = "spin-table";
69 cpu-release-addr = <0x1 0x0000fff8>;
70 };
71 cpu@301 {
72 device_type = "cpu";
73 compatible = "apm,strega", "arm,armv8";
74 reg = <0x0 0x301>;
75 enable-method = "spin-table";
76 cpu-release-addr = <0x1 0x0000fff8>;
77 };
78 };
79
80 gic: interrupt-controller@78090000 {
81 compatible = "arm,cortex-a15-gic";
82 #interrupt-cells = <3>;
83 #address-cells = <2>;
84 #size-cells = <2>;
85 interrupt-controller;
86 interrupts = <1 9 0xf04>; /* GIC Maintenence IRQ */
87 ranges = <0 0 0 0x79000000 0x0 0x800000>; /* MSI Range */
88 reg = <0x0 0x78090000 0x0 0x10000>, /* GIC Dist */
89 <0x0 0x780A0000 0x0 0x20000>, /* GIC CPU */
90 <0x0 0x780C0000 0x0 0x10000>, /* GIC VCPU Control */
91 <0x0 0x780E0000 0x0 0x20000>; /* GIC VCPU */
92 };
93
94 pmu {
95 compatible = "arm,armv8-pmuv3";
96 interrupts = <1 12 0xff04>;
97 };
98
99 timer {
100 compatible = "arm,armv8-timer";
101 interrupts = <1 0 0xff04>, /* Secure Phys IRQ */
102 <1 13 0xff04>, /* Non-secure Phys IRQ */
103 <1 14 0xff04>, /* Virt IRQ */
104 <1 15 0xff04>; /* Hyp IRQ */
105 clock-frequency = <50000000>;
106 };
107
108 soc {
109 compatible = "simple-bus";
110 #address-cells = <2>;
111 #size-cells = <2>;
112 ranges;
113
114 clocks {
115 #address-cells = <2>;
116 #size-cells = <2>;
117 ranges;
118
119 refclk: refclk {
120 compatible = "fixed-clock";
121 #clock-cells = <1>;
122 clock-frequency = <100000000>;
123 clock-output-names = "refclk";
124 };
125
126 socpll: socpll@17000120 {
127 compatible = "apm,xgene-socpll-clock";
128 #clock-cells = <1>;
129 clocks = <&refclk 0>;
130 reg = <0x0 0x17000120 0x0 0x1000>;
131 clock-output-names = "socpll";
132 };
133
134 socplldiv2: socplldiv2 {
135 compatible = "fixed-factor-clock";
136 #clock-cells = <1>;
137 clocks = <&socpll 0>;
138 clock-mult = <1>;
139 clock-div = <2>;
140 clock-output-names = "socplldiv2";
141 };
142
143 pcie0clk: pcie0clk@1f2bc000 {
144 compatible = "apm,xgene-device-clock";
145 #clock-cells = <1>;
146 clocks = <&socplldiv2 0>;
147 reg = <0x0 0x1f2bc000 0x0 0x1000>;
148 reg-names = "csr-reg";
149 clock-output-names = "pcie0clk";
150 };
151
152 xge0clk: xge0clk@1f61c000 {
153 compatible = "apm,xgene-device-clock";
154 #clock-cells = <1>;
155 clocks = <&socplldiv2 0>;
156 reg = <0x0 0x1f61c000 0x0 0x1000>;
157 reg-names = "csr-reg";
158 enable-mask = <0x3>;
159 csr-mask = <0x3>;
160 clock-output-names = "xge0clk";
161 };
162
163 xge1clk: xge1clk@1f62c000 {
164 compatible = "apm,xgene-device-clock";
165 #clock-cells = <1>;
166 clocks = <&socplldiv2 0>;
167 reg = <0x0 0x1f62c000 0x0 0x1000>;
168 reg-names = "csr-reg";
169 enable-mask = <0x3>;
170 csr-mask = <0x3>;
171 clock-output-names = "xge1clk";
172 };
173 };
174
175 scu: system-clk-controller@17000000 {
176 compatible = "apm,xgene-scu","syscon";
177 reg = <0x0 0x17000000 0x0 0x400>;
178 };
179
180 reboot: reboot@17000014 {
181 compatible = "syscon-reboot";
182 regmap = <&scu>;
183 offset = <0x14>;
184 mask = <0x1>;
185 };
186
187 serial0: serial@10600000 {
188 device_type = "serial";
189 compatible = "ns16550";
190 reg = <0 0x10600000 0x0 0x1000>;
191 reg-shift = <2>;
192 clock-frequency = <10000000>;
193 interrupt-parent = <&gic>;
194 interrupts = <0x0 0x4c 0x4>;
195 };
196
197 sata1: sata@1a000000 {
198 compatible = "apm,xgene-ahci";
199 reg = <0x0 0x1a000000 0x0 0x1000>,
200 <0x0 0x1f200000 0x0 0x1000>,
201 <0x0 0x1f20d000 0x0 0x1000>,
202 <0x0 0x1f20e000 0x0 0x1000>;
203 interrupts = <0x0 0x5a 0x4>;
204 dma-coherent;
205 };
206
207 sata2: sata@1a200000 {
208 compatible = "apm,xgene-ahci";
209 reg = <0x0 0x1a200000 0x0 0x1000>,
210 <0x0 0x1f210000 0x0 0x1000>,
211 <0x0 0x1f21d000 0x0 0x1000>,
212 <0x0 0x1f21e000 0x0 0x1000>;
213 interrupts = <0x0 0x5b 0x4>;
214 dma-coherent;
215 };
216
217 sata3: sata@1a400000 {
218 compatible = "apm,xgene-ahci";
219 reg = <0x0 0x1a400000 0x0 0x1000>,
220 <0x0 0x1f220000 0x0 0x1000>,
221 <0x0 0x1f22d000 0x0 0x1000>,
222 <0x0 0x1f22e000 0x0 0x1000>;
223 interrupts = <0x0 0x5c 0x4>;
224 dma-coherent;
225 };
226
227 sbgpio: sbgpio@17001000{
228 compatible = "apm,xgene-gpio-sb";
229 reg = <0x0 0x17001000 0x0 0x400>;
230 #gpio-cells = <2>;
231 gpio-controller;
232 interrupts = <0x0 0x28 0x1>,
233 <0x0 0x29 0x1>,
234 <0x0 0x2a 0x1>,
235 <0x0 0x2b 0x1>,
236 <0x0 0x2c 0x1>,
237 <0x0 0x2d 0x1>,
238 <0x0 0x2e 0x1>,
239 <0x0 0x2f 0x1>;
240 };
241
242 sgenet0: ethernet@1f610000 {
243 compatible = "apm,xgene2-sgenet";
244 status = "disabled";
245 reg = <0x0 0x1f610000 0x0 0x10000>,
246 <0x0 0x1f600000 0x0 0Xd100>,
247 <0x0 0x20000000 0x0 0X20000>;
248 interrupts = <0 96 4>,
249 <0 97 4>;
250 dma-coherent;
251 clocks = <&xge0clk 0>;
252 local-mac-address = [00 01 73 00 00 01];
253 phy-connection-type = "sgmii";
254 };
255
256 xgenet1: ethernet@1f620000 {
257 compatible = "apm,xgene2-xgenet";
258 status = "disabled";
259 reg = <0x0 0x1f620000 0x0 0x10000>,
260 <0x0 0x1f600000 0x0 0Xd100>,
261 <0x0 0x20000000 0x0 0X220000>;
262 interrupts = <0 108 4>,
263 <0 109 4>;
264 port-id = <1>;
265 dma-coherent;
266 clocks = <&xge1clk 0>;
267 local-mac-address = [00 01 73 00 00 02];
268 phy-connection-type = "xgmii";
269 };
270 };
271};
diff --git a/arch/arm64/boot/dts/apm/apm-storm.dtsi b/arch/arm64/boot/dts/apm/apm-storm.dtsi
index d831bc2ac204..6c5ed119934f 100644
--- a/arch/arm64/boot/dts/apm/apm-storm.dtsi
+++ b/arch/arm64/boot/dts/apm/apm-storm.dtsi
@@ -97,6 +97,11 @@
97 clock-frequency = <50000000>; 97 clock-frequency = <50000000>;
98 }; 98 };
99 99
100 pmu {
101 compatible = "apm,potenza-pmu", "arm,armv8-pmuv3";
102 interrupts = <1 12 0xff04>;
103 };
104
100 soc { 105 soc {
101 compatible = "simple-bus"; 106 compatible = "simple-bus";
102 #address-cells = <2>; 107 #address-cells = <2>;
@@ -207,6 +212,17 @@
207 clock-output-names = "xge0clk"; 212 clock-output-names = "xge0clk";
208 }; 213 };
209 214
215 xge1clk: xge1clk@1f62c000 {
216 compatible = "apm,xgene-device-clock";
217 status = "disabled";
218 #clock-cells = <1>;
219 clocks = <&socplldiv2 0>;
220 reg = <0x0 0x1f62c000 0x0 0x1000>;
221 reg-names = "csr-reg";
222 csr-mask = <0x3>;
223 clock-output-names = "xge1clk";
224 };
225
210 sataphy1clk: sataphy1clk@1f21c000 { 226 sataphy1clk: sataphy1clk@1f21c000 {
211 compatible = "apm,xgene-device-clock"; 227 compatible = "apm,xgene-device-clock";
212 #clock-cells = <1>; 228 #clock-cells = <1>;
@@ -396,6 +412,18 @@
396 0x0 0x1f 0x4>; 412 0x0 0x1f 0x4>;
397 }; 413 };
398 414
415 scu: system-clk-controller@17000000 {
416 compatible = "apm,xgene-scu","syscon";
417 reg = <0x0 0x17000000 0x0 0x400>;
418 };
419
420 reboot: reboot@17000014 {
421 compatible = "syscon-reboot";
422 regmap = <&scu>;
423 offset = <0x14>;
424 mask = <0x1>;
425 };
426
399 csw: csw@7e200000 { 427 csw: csw@7e200000 {
400 compatible = "apm,xgene-csw", "syscon"; 428 compatible = "apm,xgene-csw", "syscon";
401 reg = <0x0 0x7e200000 0x0 0x1000>; 429 reg = <0x0 0x7e200000 0x0 0x1000>;
@@ -477,6 +505,16 @@
477 reg = <0x0 0x7c600000 0x0 0x200000>; 505 reg = <0x0 0x7c600000 0x0 0x200000>;
478 pmd-controller = <3>; 506 pmd-controller = <3>;
479 }; 507 };
508
509 edacl3@7e600000 {
510 compatible = "apm,xgene-edac-l3";
511 reg = <0x0 0x7e600000 0x0 0x1000>;
512 };
513
514 edacsoc@7e930000 {
515 compatible = "apm,xgene-edac-soc-v1";
516 reg = <0x0 0x7e930000 0x0 0x1000>;
517 };
480 }; 518 };
481 519
482 pcie0: pcie@1f2b0000 { 520 pcie0: pcie@1f2b0000 {
@@ -816,6 +854,23 @@
816 phy-connection-type = "xgmii"; 854 phy-connection-type = "xgmii";
817 }; 855 };
818 856
857 xgenet1: ethernet@1f620000 {
858 compatible = "apm,xgene1-xgenet";
859 status = "disabled";
860 reg = <0x0 0x1f620000 0x0 0xd100>,
861 <0x0 0x1f600000 0x0 0Xc300>,
862 <0x0 0x18000000 0x0 0X8000>;
863 reg-names = "enet_csr", "ring_csr", "ring_cmd";
864 interrupts = <0x0 0x6C 0x4>,
865 <0x0 0x6D 0x4>;
866 port-id = <1>;
867 dma-coherent;
868 clocks = <&xge1clk 0>;
869 /* mac address will be overwritten by the bootloader */
870 local-mac-address = [00 00 00 00 00 00];
871 phy-connection-type = "xgmii";
872 };
873
819 rng: rng@10520000 { 874 rng: rng@10520000 {
820 compatible = "apm,xgene-rng"; 875 compatible = "apm,xgene-rng";
821 reg = <0x0 0x10520000 0x0 0x100>; 876 reg = <0x0 0x10520000 0x0 0x100>;
diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi
index e3ee96036eca..dd5158eb5872 100644
--- a/arch/arm64/boot/dts/arm/juno-base.dtsi
+++ b/arch/arm64/boot/dts/arm/juno-base.dtsi
@@ -17,6 +17,18 @@
17 }; 17 };
18 }; 18 };
19 19
20 mailbox: mhu@2b1f0000 {
21 compatible = "arm,mhu", "arm,primecell";
22 reg = <0x0 0x2b1f0000 0x0 0x1000>;
23 interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>,
24 <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
25 interrupt-names = "mhu_lpri_rx",
26 "mhu_hpri_rx";
27 #mbox-cells = <1>;
28 clocks = <&soc_refclk100mhz>;
29 clock-names = "apb_pclk";
30 };
31
20 gic: interrupt-controller@2c010000 { 32 gic: interrupt-controller@2c010000 {
21 compatible = "arm,gic-400", "arm,cortex-a15-gic"; 33 compatible = "arm,gic-400", "arm,cortex-a15-gic";
22 reg = <0x0 0x2c010000 0 0x1000>, 34 reg = <0x0 0x2c010000 0 0x1000>,
@@ -44,6 +56,53 @@
44 <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>; 56 <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>;
45 }; 57 };
46 58
59 sram: sram@2e000000 {
60 compatible = "arm,juno-sram-ns", "mmio-sram";
61 reg = <0x0 0x2e000000 0x0 0x8000>;
62
63 #address-cells = <1>;
64 #size-cells = <1>;
65 ranges = <0 0x0 0x2e000000 0x8000>;
66
67 cpu_scp_lpri: scp-shmem@0 {
68 compatible = "arm,juno-scp-shmem";
69 reg = <0x0 0x200>;
70 };
71
72 cpu_scp_hpri: scp-shmem@200 {
73 compatible = "arm,juno-scp-shmem";
74 reg = <0x200 0x200>;
75 };
76 };
77
78 scpi {
79 compatible = "arm,scpi";
80 mboxes = <&mailbox 1>;
81 shmem = <&cpu_scp_hpri>;
82
83 clocks {
84 compatible = "arm,scpi-clocks";
85
86 scpi_dvfs: scpi_clocks@0 {
87 compatible = "arm,scpi-dvfs-clocks";
88 #clock-cells = <1>;
89 clock-indices = <0>, <1>, <2>;
90 clock-output-names = "atlclk", "aplclk","gpuclk";
91 };
92 scpi_clk: scpi_clocks@3 {
93 compatible = "arm,scpi-variable-clocks";
94 #clock-cells = <1>;
95 clock-indices = <3>, <4>;
96 clock-output-names = "pxlclk0", "pxlclk1";
97 };
98 };
99
100 scpi_sensors0: sensors {
101 compatible = "arm,scpi-sensors";
102 #thermal-sensor-cells = <1>;
103 };
104 };
105
47 /include/ "juno-clocks.dtsi" 106 /include/ "juno-clocks.dtsi"
48 107
49 dma@7ff00000 { 108 dma@7ff00000 {
diff --git a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi
index 637e046f0e36..413f1b9ebcd4 100644
--- a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi
+++ b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi
@@ -61,48 +61,63 @@
61 61
62 button@1 { 62 button@1 {
63 debounce_interval = <50>; 63 debounce_interval = <50>;
64 wakeup = <1>; 64 wakeup-source;
65 linux,code = <116>; 65 linux,code = <116>;
66 label = "POWER"; 66 label = "POWER";
67 gpios = <&iofpga_gpio0 0 0x4>; 67 gpios = <&iofpga_gpio0 0 0x4>;
68 }; 68 };
69 button@2 { 69 button@2 {
70 debounce_interval = <50>; 70 debounce_interval = <50>;
71 wakeup = <1>; 71 wakeup-source;
72 linux,code = <102>; 72 linux,code = <102>;
73 label = "HOME"; 73 label = "HOME";
74 gpios = <&iofpga_gpio0 1 0x4>; 74 gpios = <&iofpga_gpio0 1 0x4>;
75 }; 75 };
76 button@3 { 76 button@3 {
77 debounce_interval = <50>; 77 debounce_interval = <50>;
78 wakeup = <1>; 78 wakeup-source;
79 linux,code = <152>; 79 linux,code = <152>;
80 label = "RLOCK"; 80 label = "RLOCK";
81 gpios = <&iofpga_gpio0 2 0x4>; 81 gpios = <&iofpga_gpio0 2 0x4>;
82 }; 82 };
83 button@4 { 83 button@4 {
84 debounce_interval = <50>; 84 debounce_interval = <50>;
85 wakeup = <1>; 85 wakeup-source;
86 linux,code = <115>; 86 linux,code = <115>;
87 label = "VOL+"; 87 label = "VOL+";
88 gpios = <&iofpga_gpio0 3 0x4>; 88 gpios = <&iofpga_gpio0 3 0x4>;
89 }; 89 };
90 button@5 { 90 button@5 {
91 debounce_interval = <50>; 91 debounce_interval = <50>;
92 wakeup = <1>; 92 wakeup-source;
93 linux,code = <114>; 93 linux,code = <114>;
94 label = "VOL-"; 94 label = "VOL-";
95 gpios = <&iofpga_gpio0 4 0x4>; 95 gpios = <&iofpga_gpio0 4 0x4>;
96 }; 96 };
97 button@6 { 97 button@6 {
98 debounce_interval = <50>; 98 debounce_interval = <50>;
99 wakeup = <1>; 99 wakeup-source;
100 linux,code = <99>; 100 linux,code = <99>;
101 label = "NMI"; 101 label = "NMI";
102 gpios = <&iofpga_gpio0 5 0x4>; 102 gpios = <&iofpga_gpio0 5 0x4>;
103 }; 103 };
104 }; 104 };
105 105
106 flash@0,00000000 {
107 /* 2 * 32MiB NOR Flash memory mounted on CS0 */
108 compatible = "arm,vexpress-flash", "cfi-flash";
109 linux,part-probe = "afs";
110 reg = <0 0x00000000 0x04000000>;
111 bank-width = <4>;
112 /*
113 * Unfortunately, accessing the flash disturbs
114 * the CPU idle states (suspend) and CPU
115 * hotplug of the platform. For this reason,
116 * flash hardware access is disabled by default.
117 */
118 status = "disabled";
119 };
120
106 ethernet@2,00000000 { 121 ethernet@2,00000000 {
107 compatible = "smsc,lan9118", "smsc,lan9115"; 122 compatible = "smsc,lan9118", "smsc,lan9115";
108 reg = <2 0x00000000 0x10000>; 123 reg = <2 0x00000000 0x10000>;
diff --git a/arch/arm64/boot/dts/arm/juno-r1.dts b/arch/arm64/boot/dts/arm/juno-r1.dts
index c62751153a4f..93bc3d7d51c0 100644
--- a/arch/arm64/boot/dts/arm/juno-r1.dts
+++ b/arch/arm64/boot/dts/arm/juno-r1.dts
@@ -34,12 +34,39 @@
34 #address-cells = <2>; 34 #address-cells = <2>;
35 #size-cells = <0>; 35 #size-cells = <0>;
36 36
37 cpu-map {
38 cluster0 {
39 core0 {
40 cpu = <&A57_0>;
41 };
42 core1 {
43 cpu = <&A57_1>;
44 };
45 };
46
47 cluster1 {
48 core0 {
49 cpu = <&A53_0>;
50 };
51 core1 {
52 cpu = <&A53_1>;
53 };
54 core2 {
55 cpu = <&A53_2>;
56 };
57 core3 {
58 cpu = <&A53_3>;
59 };
60 };
61 };
62
37 A57_0: cpu@0 { 63 A57_0: cpu@0 {
38 compatible = "arm,cortex-a57","arm,armv8"; 64 compatible = "arm,cortex-a57","arm,armv8";
39 reg = <0x0 0x0>; 65 reg = <0x0 0x0>;
40 device_type = "cpu"; 66 device_type = "cpu";
41 enable-method = "psci"; 67 enable-method = "psci";
42 next-level-cache = <&A57_L2>; 68 next-level-cache = <&A57_L2>;
69 clocks = <&scpi_dvfs 0>;
43 }; 70 };
44 71
45 A57_1: cpu@1 { 72 A57_1: cpu@1 {
@@ -48,6 +75,7 @@
48 device_type = "cpu"; 75 device_type = "cpu";
49 enable-method = "psci"; 76 enable-method = "psci";
50 next-level-cache = <&A57_L2>; 77 next-level-cache = <&A57_L2>;
78 clocks = <&scpi_dvfs 0>;
51 }; 79 };
52 80
53 A53_0: cpu@100 { 81 A53_0: cpu@100 {
@@ -56,6 +84,7 @@
56 device_type = "cpu"; 84 device_type = "cpu";
57 enable-method = "psci"; 85 enable-method = "psci";
58 next-level-cache = <&A53_L2>; 86 next-level-cache = <&A53_L2>;
87 clocks = <&scpi_dvfs 1>;
59 }; 88 };
60 89
61 A53_1: cpu@101 { 90 A53_1: cpu@101 {
@@ -64,6 +93,7 @@
64 device_type = "cpu"; 93 device_type = "cpu";
65 enable-method = "psci"; 94 enable-method = "psci";
66 next-level-cache = <&A53_L2>; 95 next-level-cache = <&A53_L2>;
96 clocks = <&scpi_dvfs 1>;
67 }; 97 };
68 98
69 A53_2: cpu@102 { 99 A53_2: cpu@102 {
@@ -72,6 +102,7 @@
72 device_type = "cpu"; 102 device_type = "cpu";
73 enable-method = "psci"; 103 enable-method = "psci";
74 next-level-cache = <&A53_L2>; 104 next-level-cache = <&A53_L2>;
105 clocks = <&scpi_dvfs 1>;
75 }; 106 };
76 107
77 A53_3: cpu@103 { 108 A53_3: cpu@103 {
@@ -80,6 +111,7 @@
80 device_type = "cpu"; 111 device_type = "cpu";
81 enable-method = "psci"; 112 enable-method = "psci";
82 next-level-cache = <&A53_L2>; 113 next-level-cache = <&A53_L2>;
114 clocks = <&scpi_dvfs 1>;
83 }; 115 };
84 116
85 A57_L2: l2-cache0 { 117 A57_L2: l2-cache0 {
@@ -91,17 +123,21 @@
91 }; 123 };
92 }; 124 };
93 125
94 pmu { 126 pmu_a57 {
95 compatible = "arm,armv8-pmuv3"; 127 compatible = "arm,cortex-a57-pmu";
96 interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>, 128 interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>,
97 <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>, 129 <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>;
98 <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>, 130 interrupt-affinity = <&A57_0>,
131 <&A57_1>;
132 };
133
134 pmu_a53 {
135 compatible = "arm,cortex-a53-pmu";
136 interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
99 <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>, 137 <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
100 <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>, 138 <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>,
101 <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>; 139 <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
102 interrupt-affinity = <&A57_0>, 140 interrupt-affinity = <&A53_0>,
103 <&A57_1>,
104 <&A53_0>,
105 <&A53_1>, 141 <&A53_1>,
106 <&A53_2>, 142 <&A53_2>,
107 <&A53_3>; 143 <&A53_3>;
@@ -109,6 +145,26 @@
109 145
110 #include "juno-base.dtsi" 146 #include "juno-base.dtsi"
111 147
148 pcie-controller@40000000 {
149 compatible = "arm,juno-r1-pcie", "plda,xpressrich3-axi", "pci-host-ecam-generic";
150 device_type = "pci";
151 reg = <0 0x40000000 0 0x10000000>; /* ECAM config space */
152 bus-range = <0 255>;
153 linux,pci-domain = <0>;
154 #address-cells = <3>;
155 #size-cells = <2>;
156 dma-coherent;
157 ranges = <0x01000000 0x00 0x5f800000 0x00 0x5f800000 0x0 0x00800000>,
158 <0x02000000 0x00 0x50000000 0x00 0x50000000 0x0 0x08000000>,
159 <0x42000000 0x40 0x00000000 0x40 0x00000000 0x1 0x00000000>;
160 #interrupt-cells = <1>;
161 interrupt-map-mask = <0 0 0 7>;
162 interrupt-map = <0 0 0 1 &gic 0 0 0 136 4>,
163 <0 0 0 2 &gic 0 0 0 137 4>,
164 <0 0 0 3 &gic 0 0 0 138 4>,
165 <0 0 0 4 &gic 0 0 0 139 4>;
166 msi-parent = <&v2m_0>;
167 };
112}; 168};
113 169
114&memtimer { 170&memtimer {
diff --git a/arch/arm64/boot/dts/arm/juno.dts b/arch/arm64/boot/dts/arm/juno.dts
index d7cbdd482a61..53442b5ee4ff 100644
--- a/arch/arm64/boot/dts/arm/juno.dts
+++ b/arch/arm64/boot/dts/arm/juno.dts
@@ -34,12 +34,39 @@
34 #address-cells = <2>; 34 #address-cells = <2>;
35 #size-cells = <0>; 35 #size-cells = <0>;
36 36
37 cpu-map {
38 cluster0 {
39 core0 {
40 cpu = <&A57_0>;
41 };
42 core1 {
43 cpu = <&A57_1>;
44 };
45 };
46
47 cluster1 {
48 core0 {
49 cpu = <&A53_0>;
50 };
51 core1 {
52 cpu = <&A53_1>;
53 };
54 core2 {
55 cpu = <&A53_2>;
56 };
57 core3 {
58 cpu = <&A53_3>;
59 };
60 };
61 };
62
37 A57_0: cpu@0 { 63 A57_0: cpu@0 {
38 compatible = "arm,cortex-a57","arm,armv8"; 64 compatible = "arm,cortex-a57","arm,armv8";
39 reg = <0x0 0x0>; 65 reg = <0x0 0x0>;
40 device_type = "cpu"; 66 device_type = "cpu";
41 enable-method = "psci"; 67 enable-method = "psci";
42 next-level-cache = <&A57_L2>; 68 next-level-cache = <&A57_L2>;
69 clocks = <&scpi_dvfs 0>;
43 }; 70 };
44 71
45 A57_1: cpu@1 { 72 A57_1: cpu@1 {
@@ -48,6 +75,7 @@
48 device_type = "cpu"; 75 device_type = "cpu";
49 enable-method = "psci"; 76 enable-method = "psci";
50 next-level-cache = <&A57_L2>; 77 next-level-cache = <&A57_L2>;
78 clocks = <&scpi_dvfs 0>;
51 }; 79 };
52 80
53 A53_0: cpu@100 { 81 A53_0: cpu@100 {
@@ -56,6 +84,7 @@
56 device_type = "cpu"; 84 device_type = "cpu";
57 enable-method = "psci"; 85 enable-method = "psci";
58 next-level-cache = <&A53_L2>; 86 next-level-cache = <&A53_L2>;
87 clocks = <&scpi_dvfs 1>;
59 }; 88 };
60 89
61 A53_1: cpu@101 { 90 A53_1: cpu@101 {
@@ -64,6 +93,7 @@
64 device_type = "cpu"; 93 device_type = "cpu";
65 enable-method = "psci"; 94 enable-method = "psci";
66 next-level-cache = <&A53_L2>; 95 next-level-cache = <&A53_L2>;
96 clocks = <&scpi_dvfs 1>;
67 }; 97 };
68 98
69 A53_2: cpu@102 { 99 A53_2: cpu@102 {
@@ -72,6 +102,7 @@
72 device_type = "cpu"; 102 device_type = "cpu";
73 enable-method = "psci"; 103 enable-method = "psci";
74 next-level-cache = <&A53_L2>; 104 next-level-cache = <&A53_L2>;
105 clocks = <&scpi_dvfs 1>;
75 }; 106 };
76 107
77 A53_3: cpu@103 { 108 A53_3: cpu@103 {
@@ -80,6 +111,7 @@
80 device_type = "cpu"; 111 device_type = "cpu";
81 enable-method = "psci"; 112 enable-method = "psci";
82 next-level-cache = <&A53_L2>; 113 next-level-cache = <&A53_L2>;
114 clocks = <&scpi_dvfs 1>;
83 }; 115 };
84 116
85 A57_L2: l2-cache0 { 117 A57_L2: l2-cache0 {
@@ -91,17 +123,21 @@
91 }; 123 };
92 }; 124 };
93 125
94 pmu { 126 pmu_a57 {
95 compatible = "arm,armv8-pmuv3"; 127 compatible = "arm,cortex-a57-pmu";
96 interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>, 128 interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>,
97 <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>, 129 <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>;
98 <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>, 130 interrupt-affinity = <&A57_0>,
131 <&A57_1>;
132 };
133
134 pmu_a53 {
135 compatible = "arm,cortex-a53-pmu";
136 interrupts = <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
99 <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>, 137 <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
100 <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>, 138 <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>,
101 <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>; 139 <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
102 interrupt-affinity = <&A57_0>, 140 interrupt-affinity = <&A53_0>,
103 <&A57_1>,
104 <&A53_0>,
105 <&A53_1>, 141 <&A53_1>,
106 <&A53_2>, 142 <&A53_2>,
107 <&A53_3>; 143 <&A53_3>;
diff --git a/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts b/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts
index 5b1d0181023b..bb3c26d1154d 100644
--- a/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts
+++ b/arch/arm64/boot/dts/arm/vexpress-v2f-1xv7-ca53x2.dts
@@ -186,6 +186,6 @@
186 <0 0 41 &gic GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>, 186 <0 0 41 &gic GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>,
187 <0 0 42 &gic GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>; 187 <0 0 42 &gic GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
188 188
189 /include/ "../../../../arm/boot/dts/vexpress-v2m-rs1.dtsi" 189 /include/ "vexpress-v2m-rs1.dtsi"
190 }; 190 };
191}; 191};
diff --git a/arch/arm64/boot/dts/arm/vexpress-v2m-rs1.dtsi b/arch/arm64/boot/dts/arm/vexpress-v2m-rs1.dtsi
new file mode 120000
index 000000000000..68fd0f8f1dee
--- /dev/null
+++ b/arch/arm64/boot/dts/arm/vexpress-v2m-rs1.dtsi
@@ -0,0 +1 @@
../../../../arm/boot/dts/vexpress-v2m-rs1.dtsi \ No newline at end of file
diff --git a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
index 2eef4a279131..f77ddaf21d04 100644
--- a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
@@ -586,3 +586,106 @@
586 samsung,pin-drv = <2>; 586 samsung,pin-drv = <2>;
587 }; 587 };
588}; 588};
589
590&pinctrl_bus1 {
591 gpf0: gpf0 {
592 gpio-controller;
593 #gpio-cells = <2>;
594
595 interrupt-controller;
596 #interrupt-cells = <2>;
597 };
598
599 gpf1: gpf1 {
600 gpio-controller;
601 #gpio-cells = <2>;
602
603 interrupt-controller;
604 #interrupt-cells = <2>;
605 };
606
607 gpf2: gpf2 {
608 gpio-controller;
609 #gpio-cells = <2>;
610
611 interrupt-controller;
612 #interrupt-cells = <2>;
613 };
614
615 gpf3: gpf3 {
616 gpio-controller;
617 #gpio-cells = <2>;
618
619 interrupt-controller;
620 #interrupt-cells = <2>;
621 };
622
623 gpf4: gpf4 {
624 gpio-controller;
625 #gpio-cells = <2>;
626
627 interrupt-controller;
628 #interrupt-cells = <2>;
629 };
630
631 gpf5: gpf5 {
632 gpio-controller;
633 #gpio-cells = <2>;
634
635 interrupt-controller;
636 #interrupt-cells = <2>;
637 };
638
639 gpg1: gpg1 {
640 gpio-controller;
641 #gpio-cells = <2>;
642
643 interrupt-controller;
644 #interrupt-cells = <2>;
645 };
646
647 gpg2: gpg2 {
648 gpio-controller;
649 #gpio-cells = <2>;
650
651 interrupt-controller;
652 #interrupt-cells = <2>;
653 };
654
655 gph1: gph1 {
656 gpio-controller;
657 #gpio-cells = <2>;
658
659 interrupt-controller;
660 #interrupt-cells = <2>;
661 };
662
663 gpv6: gpv6 {
664 gpio-controller;
665 #gpio-cells = <2>;
666
667 interrupt-controller;
668 #interrupt-cells = <2>;
669 };
670
671 spi5_bus: spi5-bus {
672 samsung,pins = "gpf2-0", "gpf2-1", "gpf2-2", "gpf2-3";
673 samsung,pin-function = <2>;
674 samsung,pin-pud = <3>;
675 samsung,pin-drv = <0>;
676 };
677
678 ufs_refclk_out: ufs-refclk-out {
679 samsung,pins = "gpg2-4";
680 samsung,pin-function = <2>;
681 samsung,pin-pud = <0>;
682 samsung,pin-drv = <2>;
683 };
684
685 ufs_rst_n: ufs-rst-n {
686 samsung,pins = "gph1-5";
687 samsung,pin-function = <2>;
688 samsung,pin-pud = <3>;
689 samsung,pin-drv = <0>;
690 };
691};
diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi
index d7a37c3a6b52..f9c5a549c2c0 100644
--- a/arch/arm64/boot/dts/exynos/exynos7.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi
@@ -26,6 +26,7 @@
26 pinctrl5 = &pinctrl_ese; 26 pinctrl5 = &pinctrl_ese;
27 pinctrl6 = &pinctrl_fsys0; 27 pinctrl6 = &pinctrl_fsys0;
28 pinctrl7 = &pinctrl_fsys1; 28 pinctrl7 = &pinctrl_fsys1;
29 pinctrl8 = &pinctrl_bus1;
29 }; 30 };
30 31
31 cpus { 32 cpus {
@@ -278,6 +279,12 @@
278 interrupts = <0 203 0>; 279 interrupts = <0 203 0>;
279 }; 280 };
280 281
282 pinctrl_bus1: pinctrl@14870000 {
283 compatible = "samsung,exynos7-pinctrl";
284 reg = <0x14870000 0x1000>;
285 interrupts = <0 384 0>;
286 };
287
281 hsi2c_0: hsi2c@13640000 { 288 hsi2c_0: hsi2c@13640000 {
282 compatible = "samsung,exynos7-hsi2c"; 289 compatible = "samsung,exynos7-hsi2c";
283 reg = <0x13640000 0x1000>; 290 reg = <0x13640000 0x1000>;
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 4f2de3e789ee..c4957a4aa5aa 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -1,4 +1,6 @@
1dtb-$(CONFIG_ARCH_FSL_LS2085A) += fsl-ls2085a-simu.dtb 1dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-qds.dtb
2dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-rdb.dtb
3dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-simu.dtb
2 4
3always := $(dtb-y) 5always := $(dtb-y)
4subdir-y := $(dts-dirs) 6subdir-y := $(dts-dirs)
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts
new file mode 100644
index 000000000000..4cb996d6e686
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-qds.dts
@@ -0,0 +1,204 @@
1/*
2 * Device Tree file for Freescale LS2080a QDS Board.
3 *
4 * Copyright (C) 2015, Freescale Semiconductor
5 *
6 * Bhupesh Sharma <bhupesh.sharma@freescale.com>
7 *
8 * This file is dual-licensed: you can use it either under the terms
9 * of the GPLv2 or the X11 license, at your option. Note that this dual
10 * licensing only applies to this file, and not this project as a
11 * whole.
12 *
13 * a) This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License, or (at your option) any later version.
17 *
18 * This library is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * Or, alternatively,
24 *
25 * b) Permission is hereby granted, free of charge, to any person
26 * obtaining a copy of this software and associated documentation
27 * files (the "Software"), to deal in the Software without
28 * restriction, including without limitation the rights to use,
29 * copy, modify, merge, publish, distribute, sublicense, and/or
30 * sell copies of the Software, and to permit persons to whom the
31 * Software is furnished to do so, subject to the following
32 * conditions:
33 *
34 * The above copyright notice and this permission notice shall be
35 * included in all copies or substantial portions of the Software.
36 *
37 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
38 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
39 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
40 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
41 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
42 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
43 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
44 * OTHER DEALINGS IN THE SOFTWARE.
45 */
46
47/dts-v1/;
48
49/include/ "fsl-ls2080a.dtsi"
50
51/ {
52 model = "Freescale Layerscape 2080a QDS Board";
53 compatible = "fsl,ls2080a-qds", "fsl,ls2080a";
54
55 aliases {
56 serial0 = &serial0;
57 serial1 = &serial1;
58 };
59
60};
61
62&esdhc {
63 status = "okay";
64};
65
66&ifc {
67 status = "okay";
68 #address-cells = <2>;
69 #size-cells = <1>;
70 ranges = <0x0 0x0 0x5 0x80000000 0x08000000
71 0x2 0x0 0x5 0x30000000 0x00010000
72 0x3 0x0 0x5 0x20000000 0x00010000>;
73
74 nor@0,0 {
75 #address-cells = <1>;
76 #size-cells = <1>;
77 compatible = "cfi-flash";
78 reg = <0x0 0x0 0x8000000>;
79 bank-width = <2>;
80 device-width = <1>;
81 };
82
83 nand@2,0 {
84 compatible = "fsl,ifc-nand";
85 reg = <0x2 0x0 0x10000>;
86 };
87
88 cpld@3,0 {
89 reg = <0x3 0x0 0x10000>;
90 compatible = "fsl,ls2080aqds-fpga", "fsl,fpga-qixis";
91 };
92};
93
94&i2c0 {
95 status = "okay";
96 pca9547@77 {
97 compatible = "nxp,pca9547";
98 reg = <0x77>;
99 #address-cells = <1>;
100 #size-cells = <0>;
101 i2c@0 {
102 #address-cells = <1>;
103 #size-cells = <0>;
104 reg = <0x00>;
105 rtc@68 {
106 compatible = "dallas,ds3232";
107 reg = <0x68>;
108 };
109 };
110
111 i2c@2 {
112 #address-cells = <1>;
113 #size-cells = <0>;
114 reg = <0x02>;
115
116 ina220@40 {
117 compatible = "ti,ina220";
118 reg = <0x40>;
119 shunt-resistor = <500>;
120 };
121
122 ina220@41 {
123 compatible = "ti,ina220";
124 reg = <0x41>;
125 shunt-resistor = <1000>;
126 };
127 };
128
129 i2c@3 {
130 #address-cells = <1>;
131 #size-cells = <0>;
132 reg = <0x3>;
133
134 adt7481@4c {
135 compatible = "adi,adt7461";
136 reg = <0x4c>;
137 };
138 };
139 };
140};
141
142&i2c1 {
143 status = "disabled";
144};
145
146&i2c2 {
147 status = "disabled";
148};
149
150&i2c3 {
151 status = "disabled";
152};
153
154&dspi {
155 status = "okay";
156 dflash0: n25q128a {
157 #address-cells = <1>;
158 #size-cells = <1>;
159 compatible = "st,m25p80";
160 spi-max-frequency = <3000000>;
161 reg = <0>;
162 };
163 dflash1: sst25wf040b {
164 #address-cells = <1>;
165 #size-cells = <1>;
166 compatible = "st,m25p80";
167 spi-max-frequency = <3000000>;
168 reg = <1>;
169 };
170 dflash2: en25s64 {
171 #address-cells = <1>;
172 #size-cells = <1>;
173 compatible = "st,m25p80";
174 spi-max-frequency = <3000000>;
175 reg = <2>;
176 };
177};
178
179&qspi {
180 status = "okay";
181 qflash0: s25fl008k {
182 #address-cells = <1>;
183 #size-cells = <1>;
184 compatible = "st,m25p80";
185 spi-max-frequency = <20000000>;
186 reg = <0>;
187 };
188};
189
190&sata0 {
191 status = "okay";
192};
193
194&sata1 {
195 status = "okay";
196};
197
198&usb0 {
199 status = "okay";
200};
201
202&usb1 {
203 status = "okay";
204};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
new file mode 100644
index 000000000000..e127f0baab19
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
@@ -0,0 +1,166 @@
1/*
2 * Device Tree file for Freescale LS2080a RDB Board.
3 *
4 * Copyright (C) 2015, Freescale Semiconductor
5 *
6 * Bhupesh Sharma <bhupesh.sharma@freescale.com>
7 *
8 * This file is dual-licensed: you can use it either under the terms
9 * of the GPLv2 or the X11 license, at your option. Note that this dual
10 * licensing only applies to this file, and not this project as a
11 * whole.
12 *
13 * a) This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License, or (at your option) any later version.
17 *
18 * This library is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * Or, alternatively,
24 *
25 * b) Permission is hereby granted, free of charge, to any person
26 * obtaining a copy of this software and associated documentation
27 * files (the "Software"), to deal in the Software without
28 * restriction, including without limitation the rights to use,
29 * copy, modify, merge, publish, distribute, sublicense, and/or
30 * sell copies of the Software, and to permit persons to whom the
31 * Software is furnished to do so, subject to the following
32 * conditions:
33 *
34 * The above copyright notice and this permission notice shall be
35 * included in all copies or substantial portions of the Software.
36 *
37 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
38 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
39 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
40 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
41 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
42 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
43 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
44 * OTHER DEALINGS IN THE SOFTWARE.
45 */
46
47/dts-v1/;
48
49/include/ "fsl-ls2080a.dtsi"
50
51/ {
52 model = "Freescale Layerscape 2080a RDB Board";
53 compatible = "fsl,ls2080a-rdb", "fsl,ls2080a";
54
55 aliases {
56 serial0 = &serial0;
57 serial1 = &serial1;
58 };
59};
60
61&esdhc {
62 status = "okay";
63};
64
65&ifc {
66 status = "okay";
67 #address-cells = <2>;
68 #size-cells = <1>;
69 ranges = <0x0 0x0 0x5 0x80000000 0x08000000
70 0x2 0x0 0x5 0x30000000 0x00010000
71 0x3 0x0 0x5 0x20000000 0x00010000>;
72
73 nor@0,0 {
74 #address-cells = <1>;
75 #size-cells = <1>;
76 compatible = "cfi-flash";
77 reg = <0x0 0x0 0x8000000>;
78 bank-width = <2>;
79 device-width = <1>;
80 };
81
82 nand@2,0 {
83 compatible = "fsl,ifc-nand";
84 reg = <0x2 0x0 0x10000>;
85 };
86
87 cpld@3,0 {
88 reg = <0x3 0x0 0x10000>;
89 compatible = "fsl,ls2080aqds-fpga", "fsl,fpga-qixis";
90 };
91
92};
93
94&i2c0 {
95 status = "okay";
96 pca9547@75 {
97 compatible = "nxp,pca9547";
98 reg = <0x75>;
99 #address-cells = <1>;
100 #size-cells = <0>;
101 status = "disabled";
102 i2c@1 {
103 #address-cells = <1>;
104 #size-cells = <0>;
105 reg = <0x01>;
106 rtc@68 {
107 compatible = "dallas,ds3232";
108 reg = <0x68>;
109 };
110 };
111
112 i2c@3 {
113 #address-cells = <1>;
114 #size-cells = <0>;
115 reg = <0x3>;
116
117 adt7481@4c {
118 compatible = "adi,adt7461";
119 reg = <0x4c>;
120 };
121 };
122 };
123};
124
125&i2c1 {
126 status = "disabled";
127};
128
129&i2c2 {
130 status = "disabled";
131};
132
133&i2c3 {
134 status = "disabled";
135};
136
137&dspi {
138 status = "okay";
139 dflash0: n25q512a {
140 #address-cells = <1>;
141 #size-cells = <1>;
142 compatible = "st,m25p80";
143 spi-max-frequency = <3000000>;
144 reg = <0>;
145 };
146};
147
148&qspi {
149 status = "disabled";
150};
151
152&sata0 {
153 status = "okay";
154};
155
156&sata1 {
157 status = "okay";
158};
159
160&usb0 {
161 status = "okay";
162};
163
164&usb1 {
165 status = "okay";
166};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-simu.dts
index 82e2a6fccc64..505d038078a3 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-simu.dts
@@ -1,7 +1,7 @@
1/* 1/*
2 * Device Tree file for Freescale LS2085a software Simulator model 2 * Device Tree file for Freescale LS2080a software Simulator model
3 * 3 *
4 * Copyright (C) 2014, Freescale Semiconductor 4 * Copyright (C) 2014-2015, Freescale Semiconductor
5 * 5 *
6 * Bhupesh Sharma <bhupesh.sharma@freescale.com> 6 * Bhupesh Sharma <bhupesh.sharma@freescale.com>
7 * 7 *
@@ -20,11 +20,6 @@
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details. 21 * GNU General Public License for more details.
22 * 22 *
23 * You should have received a copy of the GNU General Public
24 * License along with this library; if not, write to the Free
25 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
26 * MA 02110-1301 USA
27 *
28 * Or, alternatively, 23 * Or, alternatively,
29 * 24 *
30 * b) Permission is hereby granted, free of charge, to any person 25 * b) Permission is hereby granted, free of charge, to any person
@@ -51,11 +46,16 @@
51 46
52/dts-v1/; 47/dts-v1/;
53 48
54/include/ "fsl-ls2085a.dtsi" 49/include/ "fsl-ls2080a.dtsi"
55 50
56/ { 51/ {
57 model = "Freescale Layerscape 2085a software Simulator model"; 52 model = "Freescale Layerscape 2080a software Simulator model";
58 compatible = "fsl,ls2085a-simu", "fsl,ls2085a"; 53 compatible = "fsl,ls2080a-simu", "fsl,ls2080a";
54
55 aliases {
56 serial0 = &serial0;
57 serial1 = &serial1;
58 };
59 59
60 ethernet@2210000 { 60 ethernet@2210000 {
61 compatible = "smsc,lan91c111"; 61 compatible = "smsc,lan91c111";
@@ -63,3 +63,8 @@
63 interrupts = <0 58 0x1>; 63 interrupts = <0 58 0x1>;
64 }; 64 };
65}; 65};
66
67&ifc {
68 status = "okay";
69};
70
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
new file mode 100644
index 000000000000..e81cd48d6245
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
@@ -0,0 +1,515 @@
1/*
2 * Device Tree Include file for Freescale Layerscape-2080A family SoC.
3 *
4 * Copyright (C) 2014-2015, Freescale Semiconductor
5 *
6 * Bhupesh Sharma <bhupesh.sharma@freescale.com>
7 *
8 * This file is dual-licensed: you can use it either under the terms
9 * of the GPLv2 or the X11 license, at your option. Note that this dual
10 * licensing only applies to this file, and not this project as a
11 * whole.
12 *
13 * a) This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License, or (at your option) any later version.
17 *
18 * This library is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * Or, alternatively,
24 *
25 * b) Permission is hereby granted, free of charge, to any person
26 * obtaining a copy of this software and associated documentation
27 * files (the "Software"), to deal in the Software without
28 * restriction, including without limitation the rights to use,
29 * copy, modify, merge, publish, distribute, sublicense, and/or
30 * sell copies of the Software, and to permit persons to whom the
31 * Software is furnished to do so, subject to the following
32 * conditions:
33 *
34 * The above copyright notice and this permission notice shall be
35 * included in all copies or substantial portions of the Software.
36 *
37 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
38 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
39 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
40 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
41 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
42 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
43 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
44 * OTHER DEALINGS IN THE SOFTWARE.
45 */
46
47/ {
48 compatible = "fsl,ls2080a";
49 interrupt-parent = <&gic>;
50 #address-cells = <2>;
51 #size-cells = <2>;
52
53 cpus {
54 #address-cells = <2>;
55 #size-cells = <0>;
56
57 /*
58 * We expect the enable-method for cpu's to be "psci", but this
59 * is dependent on the SoC FW, which will fill this in.
60 *
61 * Currently supported enable-method is psci v0.2
62 */
63
64 /* We have 4 clusters having 2 Cortex-A57 cores each */
65 cpu@0 {
66 device_type = "cpu";
67 compatible = "arm,cortex-a57";
68 reg = <0x0 0x0>;
69 clocks = <&clockgen 1 0>;
70 };
71
72 cpu@1 {
73 device_type = "cpu";
74 compatible = "arm,cortex-a57";
75 reg = <0x0 0x1>;
76 clocks = <&clockgen 1 0>;
77 };
78
79 cpu@100 {
80 device_type = "cpu";
81 compatible = "arm,cortex-a57";
82 reg = <0x0 0x100>;
83 clocks = <&clockgen 1 1>;
84 };
85
86 cpu@101 {
87 device_type = "cpu";
88 compatible = "arm,cortex-a57";
89 reg = <0x0 0x101>;
90 clocks = <&clockgen 1 1>;
91 };
92
93 cpu@200 {
94 device_type = "cpu";
95 compatible = "arm,cortex-a57";
96 reg = <0x0 0x200>;
97 clocks = <&clockgen 1 2>;
98 };
99
100 cpu@201 {
101 device_type = "cpu";
102 compatible = "arm,cortex-a57";
103 reg = <0x0 0x201>;
104 clocks = <&clockgen 1 2>;
105 };
106
107 cpu@300 {
108 device_type = "cpu";
109 compatible = "arm,cortex-a57";
110 reg = <0x0 0x300>;
111 clocks = <&clockgen 1 3>;
112 };
113
114 cpu@301 {
115 device_type = "cpu";
116 compatible = "arm,cortex-a57";
117 reg = <0x0 0x301>;
118 clocks = <&clockgen 1 3>;
119 };
120 };
121
122 memory@80000000 {
123 device_type = "memory";
124 reg = <0x00000000 0x80000000 0 0x80000000>;
125 /* DRAM space - 1, size : 2 GB DRAM */
126 };
127
128 sysclk: sysclk {
129 compatible = "fixed-clock";
130 #clock-cells = <0>;
131 clock-frequency = <100000000>;
132 clock-output-names = "sysclk";
133 };
134
135 gic: interrupt-controller@6000000 {
136 compatible = "arm,gic-v3";
137 reg = <0x0 0x06000000 0 0x10000>, /* GIC Dist */
138 <0x0 0x06100000 0 0x100000>, /* GICR (RD_base + SGI_base) */
139 <0x0 0x0c0c0000 0 0x2000>, /* GICC */
140 <0x0 0x0c0d0000 0 0x1000>, /* GICH */
141 <0x0 0x0c0e0000 0 0x20000>; /* GICV */
142 #interrupt-cells = <3>;
143 #address-cells = <2>;
144 #size-cells = <2>;
145 ranges;
146 interrupt-controller;
147 interrupts = <1 9 0x4>;
148
149 its: gic-its@6020000 {
150 compatible = "arm,gic-v3-its";
151 msi-controller;
152 reg = <0x0 0x6020000 0 0x20000>;
153 };
154 };
155
156 timer {
157 compatible = "arm,armv8-timer";
158 interrupts = <1 13 0x8>, /* Physical Secure PPI, active-low */
159 <1 14 0x8>, /* Physical Non-Secure PPI, active-low */
160 <1 11 0x8>, /* Virtual PPI, active-low */
161 <1 10 0x8>; /* Hypervisor PPI, active-low */
162 };
163
164 pmu {
165 compatible = "arm,armv8-pmuv3";
166 interrupts = <1 7 0x8>; /* PMU PPI, Level low type */
167 };
168
169 soc {
170 compatible = "simple-bus";
171 #address-cells = <2>;
172 #size-cells = <2>;
173 ranges;
174
175 clockgen: clocking@1300000 {
176 compatible = "fsl,ls2080a-clockgen";
177 reg = <0 0x1300000 0 0xa0000>;
178 #clock-cells = <2>;
179 clocks = <&sysclk>;
180 };
181
182 serial0: serial@21c0500 {
183 compatible = "fsl,ns16550", "ns16550a";
184 reg = <0x0 0x21c0500 0x0 0x100>;
185 clocks = <&clockgen 4 3>;
186 interrupts = <0 32 0x4>; /* Level high type */
187 };
188
189 serial1: serial@21c0600 {
190 compatible = "fsl,ns16550", "ns16550a";
191 reg = <0x0 0x21c0600 0x0 0x100>;
192 clocks = <&clockgen 4 3>;
193 interrupts = <0 32 0x4>; /* Level high type */
194 };
195
196 fsl_mc: fsl-mc@80c000000 {
197 compatible = "fsl,qoriq-mc";
198 reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */
199 <0x00000000 0x08340000 0 0x40000>; /* MC control reg */
200 };
201
202 smmu: iommu@5000000 {
203 compatible = "arm,mmu-500";
204 reg = <0 0x5000000 0 0x800000>;
205 #global-interrupts = <12>;
206 interrupts = <0 13 4>, /* global secure fault */
207 <0 14 4>, /* combined secure interrupt */
208 <0 15 4>, /* global non-secure fault */
209 <0 16 4>, /* combined non-secure interrupt */
210 /* performance counter interrupts 0-7 */
211 <0 211 4>, <0 212 4>,
212 <0 213 4>, <0 214 4>,
213 <0 215 4>, <0 216 4>,
214 <0 217 4>, <0 218 4>,
215 /* per context interrupt, 64 interrupts */
216 <0 146 4>, <0 147 4>,
217 <0 148 4>, <0 149 4>,
218 <0 150 4>, <0 151 4>,
219 <0 152 4>, <0 153 4>,
220 <0 154 4>, <0 155 4>,
221 <0 156 4>, <0 157 4>,
222 <0 158 4>, <0 159 4>,
223 <0 160 4>, <0 161 4>,
224 <0 162 4>, <0 163 4>,
225 <0 164 4>, <0 165 4>,
226 <0 166 4>, <0 167 4>,
227 <0 168 4>, <0 169 4>,
228 <0 170 4>, <0 171 4>,
229 <0 172 4>, <0 173 4>,
230 <0 174 4>, <0 175 4>,
231 <0 176 4>, <0 177 4>,
232 <0 178 4>, <0 179 4>,
233 <0 180 4>, <0 181 4>,
234 <0 182 4>, <0 183 4>,
235 <0 184 4>, <0 185 4>,
236 <0 186 4>, <0 187 4>,
237 <0 188 4>, <0 189 4>,
238 <0 190 4>, <0 191 4>,
239 <0 192 4>, <0 193 4>,
240 <0 194 4>, <0 195 4>,
241 <0 196 4>, <0 197 4>,
242 <0 198 4>, <0 199 4>,
243 <0 200 4>, <0 201 4>,
244 <0 202 4>, <0 203 4>,
245 <0 204 4>, <0 205 4>,
246 <0 206 4>, <0 207 4>,
247 <0 208 4>, <0 209 4>;
248 mmu-masters = <&fsl_mc 0x300 0>;
249 };
250
251 dspi: dspi@2100000 {
252 status = "disabled";
253 compatible = "fsl,vf610-dspi";
254 #address-cells = <1>;
255 #size-cells = <0>;
256 reg = <0x0 0x2100000 0x0 0x10000>;
257 interrupts = <0 26 0x4>; /* Level high type */
258 clocks = <&clockgen 4 3>;
259 clock-names = "dspi";
260 spi-num-chipselects = <5>;
261 bus-num = <0>;
262 };
263
264 esdhc: esdhc@2140000 {
265 status = "disabled";
266 compatible = "fsl,ls2080a-esdhc", "fsl,esdhc";
267 reg = <0x0 0x2140000 0x0 0x10000>;
268 interrupts = <0 28 0x4>; /* Level high type */
269 clock-frequency = <0>; /* Updated by bootloader */
270 voltage-ranges = <1800 1800 3300 3300>;
271 sdhci,auto-cmd12;
272 bus-width = <4>;
273 };
274
275 gpio0: gpio@2300000 {
276 compatible = "fsl,qoriq-gpio";
277 reg = <0x0 0x2300000 0x0 0x10000>;
278 interrupts = <0 36 0x4>; /* Level high type */
279 gpio-controller;
280 #gpio-cells = <2>;
281 interrupt-controller;
282 #interrupt-cells = <2>;
283 };
284
285 gpio1: gpio@2310000 {
286 compatible = "fsl,qoriq-gpio";
287 reg = <0x0 0x2310000 0x0 0x10000>;
288 interrupts = <0 36 0x4>; /* Level high type */
289 gpio-controller;
290 #gpio-cells = <2>;
291 interrupt-controller;
292 #interrupt-cells = <2>;
293 };
294
295 gpio2: gpio@2320000 {
296 compatible = "fsl,qoriq-gpio";
297 reg = <0x0 0x2320000 0x0 0x10000>;
298 interrupts = <0 37 0x4>; /* Level high type */
299 gpio-controller;
300 #gpio-cells = <2>;
301 interrupt-controller;
302 #interrupt-cells = <2>;
303 };
304
305 gpio3: gpio@2330000 {
306 compatible = "fsl,qoriq-gpio";
307 reg = <0x0 0x2330000 0x0 0x10000>;
308 interrupts = <0 37 0x4>; /* Level high type */
309 gpio-controller;
310 #gpio-cells = <2>;
311 interrupt-controller;
312 #interrupt-cells = <2>;
313 };
314
315 i2c0: i2c@2000000 {
316 status = "disabled";
317 compatible = "fsl,vf610-i2c";
318 #address-cells = <1>;
319 #size-cells = <0>;
320 reg = <0x0 0x2000000 0x0 0x10000>;
321 interrupts = <0 34 0x4>; /* Level high type */
322 clock-names = "i2c";
323 clocks = <&clockgen 4 3>;
324 };
325
326 i2c1: i2c@2010000 {
327 status = "disabled";
328 compatible = "fsl,vf610-i2c";
329 #address-cells = <1>;
330 #size-cells = <0>;
331 reg = <0x0 0x2010000 0x0 0x10000>;
332 interrupts = <0 34 0x4>; /* Level high type */
333 clock-names = "i2c";
334 clocks = <&clockgen 4 3>;
335 };
336
337 i2c2: i2c@2020000 {
338 status = "disabled";
339 compatible = "fsl,vf610-i2c";
340 #address-cells = <1>;
341 #size-cells = <0>;
342 reg = <0x0 0x2020000 0x0 0x10000>;
343 interrupts = <0 35 0x4>; /* Level high type */
344 clock-names = "i2c";
345 clocks = <&clockgen 4 3>;
346 };
347
348 i2c3: i2c@2030000 {
349 status = "disabled";
350 compatible = "fsl,vf610-i2c";
351 #address-cells = <1>;
352 #size-cells = <0>;
353 reg = <0x0 0x2030000 0x0 0x10000>;
354 interrupts = <0 35 0x4>; /* Level high type */
355 clock-names = "i2c";
356 clocks = <&clockgen 4 3>;
357 };
358
359 ifc: ifc@2240000 {
360 compatible = "fsl,ifc", "simple-bus";
361 reg = <0x0 0x2240000 0x0 0x20000>;
362 interrupts = <0 21 0x4>; /* Level high type */
363 little-endian;
364 #address-cells = <2>;
365 #size-cells = <1>;
366
367 ranges = <0 0 0x5 0x80000000 0x08000000
368 2 0 0x5 0x30000000 0x00010000
369 3 0 0x5 0x20000000 0x00010000>;
370 };
371
372 qspi: quadspi@20c0000 {
373 status = "disabled";
374 compatible = "fsl,vf610-qspi";
375 #address-cells = <1>;
376 #size-cells = <0>;
377 reg = <0x0 0x20c0000 0x0 0x10000>,
378 <0x0 0x20000000 0x0 0x10000000>;
379 reg-names = "QuadSPI", "QuadSPI-memory";
380 interrupts = <0 25 0x4>; /* Level high type */
381 clocks = <&clockgen 4 3>, <&clockgen 4 3>;
382 clock-names = "qspi_en", "qspi";
383 };
384
385 pcie@3400000 {
386 compatible = "fsl,ls2080a-pcie", "snps,dw-pcie";
387 reg = <0x00 0x03400000 0x0 0x00100000 /* controller registers */
388 0x10 0x00000000 0x0 0x00002000>; /* configuration space */
389 reg-names = "regs", "config";
390 interrupts = <0 108 0x4>; /* Level high type */
391 interrupt-names = "intr";
392 #address-cells = <3>;
393 #size-cells = <2>;
394 device_type = "pci";
395 num-lanes = <4>;
396 bus-range = <0x0 0xff>;
397 ranges = <0x81000000 0x0 0x00000000 0x10 0x00010000 0x0 0x00010000 /* downstream I/O */
398 0x82000000 0x0 0x40000000 0x10 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */
399 msi-parent = <&its>;
400 #interrupt-cells = <1>;
401 interrupt-map-mask = <0 0 0 7>;
402 interrupt-map = <0000 0 0 1 &gic 0 0 0 109 4>,
403 <0000 0 0 2 &gic 0 0 0 110 4>,
404 <0000 0 0 3 &gic 0 0 0 111 4>,
405 <0000 0 0 4 &gic 0 0 0 112 4>;
406 };
407
408 pcie@3500000 {
409 compatible = "fsl,ls2080a-pcie", "snps,dw-pcie";
410 reg = <0x00 0x03500000 0x0 0x00100000 /* controller registers */
411 0x12 0x00000000 0x0 0x00002000>; /* configuration space */
412 reg-names = "regs", "config";
413 interrupts = <0 113 0x4>; /* Level high type */
414 interrupt-names = "intr";
415 #address-cells = <3>;
416 #size-cells = <2>;
417 device_type = "pci";
418 num-lanes = <4>;
419 bus-range = <0x0 0xff>;
420 ranges = <0x81000000 0x0 0x00000000 0x12 0x00010000 0x0 0x00010000 /* downstream I/O */
421 0x82000000 0x0 0x40000000 0x12 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */
422 msi-parent = <&its>;
423 #interrupt-cells = <1>;
424 interrupt-map-mask = <0 0 0 7>;
425 interrupt-map = <0000 0 0 1 &gic 0 0 0 114 4>,
426 <0000 0 0 2 &gic 0 0 0 115 4>,
427 <0000 0 0 3 &gic 0 0 0 116 4>,
428 <0000 0 0 4 &gic 0 0 0 117 4>;
429 };
430
431 pcie@3600000 {
432 compatible = "fsl,ls2080a-pcie", "snps,dw-pcie";
433 reg = <0x00 0x03600000 0x0 0x00100000 /* controller registers */
434 0x14 0x00000000 0x0 0x00002000>; /* configuration space */
435 reg-names = "regs", "config";
436 interrupts = <0 118 0x4>; /* Level high type */
437 interrupt-names = "intr";
438 #address-cells = <3>;
439 #size-cells = <2>;
440 device_type = "pci";
441 num-lanes = <8>;
442 bus-range = <0x0 0xff>;
443 ranges = <0x81000000 0x0 0x00000000 0x14 0x00010000 0x0 0x00010000 /* downstream I/O */
444 0x82000000 0x0 0x40000000 0x14 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */
445 msi-parent = <&its>;
446 #interrupt-cells = <1>;
447 interrupt-map-mask = <0 0 0 7>;
448 interrupt-map = <0000 0 0 1 &gic 0 0 0 119 4>,
449 <0000 0 0 2 &gic 0 0 0 120 4>,
450 <0000 0 0 3 &gic 0 0 0 121 4>,
451 <0000 0 0 4 &gic 0 0 0 122 4>;
452 };
453
454 pcie@3700000 {
455 compatible = "fsl,ls2080a-pcie", "snps,dw-pcie";
456 reg = <0x00 0x03700000 0x0 0x00100000 /* controller registers */
457 0x16 0x00000000 0x0 0x00002000>; /* configuration space */
458 reg-names = "regs", "config";
459 interrupts = <0 123 0x4>; /* Level high type */
460 interrupt-names = "intr";
461 #address-cells = <3>;
462 #size-cells = <2>;
463 device_type = "pci";
464 num-lanes = <4>;
465 bus-range = <0x0 0xff>;
466 ranges = <0x81000000 0x0 0x00000000 0x16 0x00010000 0x0 0x00010000 /* downstream I/O */
467 0x82000000 0x0 0x40000000 0x16 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */
468 msi-parent = <&its>;
469 #interrupt-cells = <1>;
470 interrupt-map-mask = <0 0 0 7>;
471 interrupt-map = <0000 0 0 1 &gic 0 0 0 124 4>,
472 <0000 0 0 2 &gic 0 0 0 125 4>,
473 <0000 0 0 3 &gic 0 0 0 126 4>,
474 <0000 0 0 4 &gic 0 0 0 127 4>;
475 };
476
477 sata0: sata@3200000 {
478 status = "disabled";
479 compatible = "fsl,ls2080a-ahci";
480 reg = <0x0 0x3200000 0x0 0x10000>;
481 interrupts = <0 133 0x4>; /* Level high type */
482 clocks = <&clockgen 4 3>;
483 };
484
485 sata1: sata@3210000 {
486 status = "disabled";
487 compatible = "fsl,ls2080a-ahci";
488 reg = <0x0 0x3210000 0x0 0x10000>;
489 interrupts = <0 136 0x4>; /* Level high type */
490 clocks = <&clockgen 4 3>;
491 };
492
493 usb0: usb3@3100000 {
494 status = "disabled";
495 compatible = "snps,dwc3";
496 reg = <0x0 0x3100000 0x0 0x10000>;
497 interrupts = <0 80 0x4>; /* Level high type */
498 dr_mode = "host";
499 };
500
501 usb1: usb3@3110000 {
502 status = "disabled";
503 compatible = "snps,dwc3";
504 reg = <0x0 0x3110000 0x0 0x10000>;
505 interrupts = <0 81 0x4>; /* Level high type */
506 dr_mode = "host";
507 };
508
509 ccn@4000000 {
510 compatible = "arm,ccn-504";
511 reg = <0x0 0x04000000 0x0 0x01000000>;
512 interrupts = <0 12 4>;
513 };
514 };
515};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi
deleted file mode 100644
index e281ceb338c3..000000000000
--- a/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi
+++ /dev/null
@@ -1,163 +0,0 @@
1/*
2 * Device Tree Include file for Freescale Layerscape-2085A family SoC.
3 *
4 * Copyright (C) 2014, Freescale Semiconductor
5 *
6 * Bhupesh Sharma <bhupesh.sharma@freescale.com>
7 *
8 * This file is dual-licensed: you can use it either under the terms
9 * of the GPLv2 or the X11 license, at your option. Note that this dual
10 * licensing only applies to this file, and not this project as a
11 * whole.
12 *
13 * a) This library is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License, or (at your option) any later version.
17 *
18 * This library is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public
24 * License along with this library; if not, write to the Free
25 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
26 * MA 02110-1301 USA
27 *
28 * Or, alternatively,
29 *
30 * b) Permission is hereby granted, free of charge, to any person
31 * obtaining a copy of this software and associated documentation
32 * files (the "Software"), to deal in the Software without
33 * restriction, including without limitation the rights to use,
34 * copy, modify, merge, publish, distribute, sublicense, and/or
35 * sell copies of the Software, and to permit persons to whom the
36 * Software is furnished to do so, subject to the following
37 * conditions:
38 *
39 * The above copyright notice and this permission notice shall be
40 * included in all copies or substantial portions of the Software.
41 *
42 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
43 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
44 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
45 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
46 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
47 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
48 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
49 * OTHER DEALINGS IN THE SOFTWARE.
50 */
51
52/ {
53 compatible = "fsl,ls2085a";
54 interrupt-parent = <&gic>;
55 #address-cells = <2>;
56 #size-cells = <2>;
57
58 cpus {
59 #address-cells = <2>;
60 #size-cells = <0>;
61
62 /*
63 * We expect the enable-method for cpu's to be "psci", but this
64 * is dependent on the SoC FW, which will fill this in.
65 *
66 * Currently supported enable-method is psci v0.2
67 */
68
69 /* We have 4 clusters having 2 Cortex-A57 cores each */
70 cpu@0 {
71 device_type = "cpu";
72 compatible = "arm,cortex-a57";
73 reg = <0x0 0x0>;
74 };
75
76 cpu@1 {
77 device_type = "cpu";
78 compatible = "arm,cortex-a57";
79 reg = <0x0 0x1>;
80 };
81
82 cpu@100 {
83 device_type = "cpu";
84 compatible = "arm,cortex-a57";
85 reg = <0x0 0x100>;
86 };
87
88 cpu@101 {
89 device_type = "cpu";
90 compatible = "arm,cortex-a57";
91 reg = <0x0 0x101>;
92 };
93
94 cpu@200 {
95 device_type = "cpu";
96 compatible = "arm,cortex-a57";
97 reg = <0x0 0x200>;
98 };
99
100 cpu@201 {
101 device_type = "cpu";
102 compatible = "arm,cortex-a57";
103 reg = <0x0 0x201>;
104 };
105
106 cpu@300 {
107 device_type = "cpu";
108 compatible = "arm,cortex-a57";
109 reg = <0x0 0x300>;
110 };
111
112 cpu@301 {
113 device_type = "cpu";
114 compatible = "arm,cortex-a57";
115 reg = <0x0 0x301>;
116 };
117 };
118
119 memory@80000000 {
120 device_type = "memory";
121 reg = <0x00000000 0x80000000 0 0x80000000>;
122 /* DRAM space - 1, size : 2 GB DRAM */
123 };
124
125 gic: interrupt-controller@6000000 {
126 compatible = "arm,gic-v3";
127 reg = <0x0 0x06000000 0 0x10000>, /* GIC Dist */
128 <0x0 0x06100000 0 0x100000>; /* GICR (RD_base + SGI_base) */
129 #interrupt-cells = <3>;
130 interrupt-controller;
131 interrupts = <1 9 0x4>;
132 };
133
134 timer {
135 compatible = "arm,armv8-timer";
136 interrupts = <1 13 0x8>, /* Physical Secure PPI, active-low */
137 <1 14 0x8>, /* Physical Non-Secure PPI, active-low */
138 <1 11 0x8>, /* Virtual PPI, active-low */
139 <1 10 0x8>; /* Hypervisor PPI, active-low */
140 };
141
142 serial0: serial@21c0500 {
143 device_type = "serial";
144 compatible = "fsl,ns16550", "ns16550a";
145 reg = <0x0 0x21c0500 0x0 0x100>;
146 clock-frequency = <0>; /* Updated by bootloader */
147 interrupts = <0 32 0x1>; /* edge triggered */
148 };
149
150 serial1: serial@21c0600 {
151 device_type = "serial";
152 compatible = "fsl,ns16550", "ns16550a";
153 reg = <0x0 0x21c0600 0x0 0x100>;
154 clock-frequency = <0>; /* Updated by bootloader */
155 interrupts = <0 32 0x1>; /* edge triggered */
156 };
157
158 fsl_mc: fsl-mc@80c000000 {
159 compatible = "fsl,qoriq-mc";
160 reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */
161 <0x00000000 0x08340000 0 0x40000>; /* MC control reg */
162 };
163};
diff --git a/arch/arm64/boot/dts/hisilicon/Makefile b/arch/arm64/boot/dts/hisilicon/Makefile
index fa81a6ee6473..cd158b80e29b 100644
--- a/arch/arm64/boot/dts/hisilicon/Makefile
+++ b/arch/arm64/boot/dts/hisilicon/Makefile
@@ -1,4 +1,4 @@
1dtb-$(CONFIG_ARCH_HISI) += hi6220-hikey.dtb 1dtb-$(CONFIG_ARCH_HISI) += hi6220-hikey.dtb hip05-d02.dtb
2 2
3always := $(dtb-y) 3always := $(dtb-y)
4subdir-y := $(dts-dirs) 4subdir-y := $(dts-dirs)
diff --git a/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts b/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts
index e36a539468a5..8d43a0fce522 100644
--- a/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts
+++ b/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts
@@ -17,11 +17,14 @@
17 compatible = "hisilicon,hi6220-hikey", "hisilicon,hi6220"; 17 compatible = "hisilicon,hi6220-hikey", "hisilicon,hi6220";
18 18
19 aliases { 19 aliases {
20 serial0 = &uart0; 20 serial0 = &uart0; /* On board UART0 */
21 serial1 = &uart1; /* BT UART */
22 serial2 = &uart2; /* LS Expansion UART0 */
23 serial3 = &uart3; /* LS Expansion UART1 */
21 }; 24 };
22 25
23 chosen { 26 chosen {
24 stdout-path = "serial0:115200n8"; 27 stdout-path = "serial3:115200n8";
25 }; 28 };
26 29
27 memory@0 { 30 memory@0 {
diff --git a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
index 3f03380815b6..82d2488a0e86 100644
--- a/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hi6220.dtsi
@@ -5,6 +5,7 @@
5 */ 5 */
6 6
7#include <dt-bindings/interrupt-controller/arm-gic.h> 7#include <dt-bindings/interrupt-controller/arm-gic.h>
8#include <dt-bindings/clock/hi6220-clock.h>
8 9
9/ { 10/ {
10 compatible = "hisilicon,hi6220"; 11 compatible = "hisilicon,hi6220";
@@ -164,8 +165,48 @@
164 compatible = "arm,pl011", "arm,primecell"; 165 compatible = "arm,pl011", "arm,primecell";
165 reg = <0x0 0xf8015000 0x0 0x1000>; 166 reg = <0x0 0xf8015000 0x0 0x1000>;
166 interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>; 167 interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
167 clocks = <&ao_ctrl 36>, <&ao_ctrl 36>; 168 clocks = <&ao_ctrl HI6220_UART0_PCLK>,
169 <&ao_ctrl HI6220_UART0_PCLK>;
168 clock-names = "uartclk", "apb_pclk"; 170 clock-names = "uartclk", "apb_pclk";
169 }; 171 };
172
173 uart1: uart@f7111000 {
174 compatible = "arm,pl011", "arm,primecell";
175 reg = <0x0 0xf7111000 0x0 0x1000>;
176 interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
177 clocks = <&sys_ctrl HI6220_UART1_PCLK>,
178 <&sys_ctrl HI6220_UART1_PCLK>;
179 clock-names = "uartclk", "apb_pclk";
180 status = "disabled";
181 };
182
183 uart2: uart@f7112000 {
184 compatible = "arm,pl011", "arm,primecell";
185 reg = <0x0 0xf7112000 0x0 0x1000>;
186 interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
187 clocks = <&sys_ctrl HI6220_UART2_PCLK>,
188 <&sys_ctrl HI6220_UART2_PCLK>;
189 clock-names = "uartclk", "apb_pclk";
190 status = "disabled";
191 };
192
193 uart3: uart@f7113000 {
194 compatible = "arm,pl011", "arm,primecell";
195 reg = <0x0 0xf7113000 0x0 0x1000>;
196 interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
197 clocks = <&sys_ctrl HI6220_UART3_PCLK>,
198 <&sys_ctrl HI6220_UART3_PCLK>;
199 clock-names = "uartclk", "apb_pclk";
200 };
201
202 uart4: uart@f7114000 {
203 compatible = "arm,pl011", "arm,primecell";
204 reg = <0x0 0xf7114000 0x0 0x1000>;
205 interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
206 clocks = <&sys_ctrl HI6220_UART4_PCLK>,
207 <&sys_ctrl HI6220_UART4_PCLK>;
208 clock-names = "uartclk", "apb_pclk";
209 status = "disabled";
210 };
170 }; 211 };
171}; 212};
diff --git a/arch/arm64/boot/dts/hisilicon/hip05-d02.dts b/arch/arm64/boot/dts/hisilicon/hip05-d02.dts
new file mode 100644
index 000000000000..ae34e250456f
--- /dev/null
+++ b/arch/arm64/boot/dts/hisilicon/hip05-d02.dts
@@ -0,0 +1,36 @@
1/**
2 * dts file for Hisilicon D02 Development Board
3 *
4 * Copyright (C) 2014,2015 Hisilicon Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * publishhed by the Free Software Foundation.
9 *
10 */
11
12/dts-v1/;
13
14#include "hip05.dtsi"
15
16/ {
17 model = "Hisilicon Hip05 D02 Development Board";
18 compatible = "hisilicon,hip05-d02";
19
20 memory@00000000 {
21 device_type = "memory";
22 reg = <0x0 0x00000000 0x0 0x80000000>;
23 };
24
25 aliases {
26 serial0 = &uart0;
27 };
28
29 chosen {
30 stdout-path = "serial0:115200n8";
31 };
32};
33
34&uart0 {
35 status = "ok";
36};
diff --git a/arch/arm64/boot/dts/hisilicon/hip05.dtsi b/arch/arm64/boot/dts/hisilicon/hip05.dtsi
new file mode 100644
index 000000000000..4ff16d016e34
--- /dev/null
+++ b/arch/arm64/boot/dts/hisilicon/hip05.dtsi
@@ -0,0 +1,271 @@
1/**
2 * dts file for Hisilicon D02 Development Board
3 *
4 * Copyright (C) 2014,2015 Hisilicon Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * publishhed by the Free Software Foundation.
9 *
10 */
11
12#include <dt-bindings/interrupt-controller/arm-gic.h>
13
14/ {
15 compatible = "hisilicon,hip05-d02";
16 interrupt-parent = <&gic>;
17 #address-cells = <2>;
18 #size-cells = <2>;
19
20 psci {
21 compatible = "arm,psci-0.2";
22 method = "smc";
23 };
24
25 cpus {
26 #address-cells = <1>;
27 #size-cells = <0>;
28
29 cpu-map {
30 cluster0 {
31 core0 {
32 cpu = <&cpu0>;
33 };
34 core1 {
35 cpu = <&cpu1>;
36 };
37 core2 {
38 cpu = <&cpu2>;
39 };
40 core3 {
41 cpu = <&cpu3>;
42 };
43 };
44 cluster1 {
45 core0 {
46 cpu = <&cpu4>;
47 };
48 core1 {
49 cpu = <&cpu5>;
50 };
51 core2 {
52 cpu = <&cpu6>;
53 };
54 core3 {
55 cpu = <&cpu7>;
56 };
57 };
58 cluster2 {
59 core0 {
60 cpu = <&cpu8>;
61 };
62 core1 {
63 cpu = <&cpu9>;
64 };
65 core2 {
66 cpu = <&cpu10>;
67 };
68 core3 {
69 cpu = <&cpu11>;
70 };
71 };
72 cluster3 {
73 core0 {
74 cpu = <&cpu12>;
75 };
76 core1 {
77 cpu = <&cpu13>;
78 };
79 core2 {
80 cpu = <&cpu14>;
81 };
82 core3 {
83 cpu = <&cpu15>;
84 };
85 };
86 };
87
88 cpu0: cpu@20000 {
89 device_type = "cpu";
90 compatible = "arm,cortex-a57", "arm,armv8";
91 reg = <0x20000>;
92 enable-method = "psci";
93 };
94
95 cpu1: cpu@20001 {
96 device_type = "cpu";
97 compatible = "arm,cortex-a57", "arm,armv8";
98 reg = <0x20001>;
99 enable-method = "psci";
100 };
101
102 cpu2: cpu@20002 {
103 device_type = "cpu";
104 compatible = "arm,cortex-a57", "arm,armv8";
105 reg = <0x20002>;
106 enable-method = "psci";
107 };
108
109 cpu3: cpu@20003 {
110 device_type = "cpu";
111 compatible = "arm,cortex-a57", "arm,armv8";
112 reg = <0x20003>;
113 enable-method = "psci";
114 };
115
116 cpu4: cpu@20100 {
117 device_type = "cpu";
118 compatible = "arm,cortex-a57", "arm,armv8";
119 reg = <0x20100>;
120 enable-method = "psci";
121 };
122
123 cpu5: cpu@20101 {
124 device_type = "cpu";
125 compatible = "arm,cortex-a57", "arm,armv8";
126 reg = <0x20101>;
127 enable-method = "psci";
128 };
129
130 cpu6: cpu@20102 {
131 device_type = "cpu";
132 compatible = "arm,cortex-a57", "arm,armv8";
133 reg = <0x20102>;
134 enable-method = "psci";
135 };
136
137 cpu7: cpu@20103 {
138 device_type = "cpu";
139 compatible = "arm,cortex-a57", "arm,armv8";
140 reg = <0x20103>;
141 enable-method = "psci";
142 };
143
144 cpu8: cpu@20200 {
145 device_type = "cpu";
146 compatible = "arm,cortex-a57", "arm,armv8";
147 reg = <0x20200>;
148 enable-method = "psci";
149 };
150
151 cpu9: cpu@20201 {
152 device_type = "cpu";
153 compatible = "arm,cortex-a57", "arm,armv8";
154 reg = <0x20201>;
155 enable-method = "psci";
156 };
157
158 cpu10: cpu@20202 {
159 device_type = "cpu";
160 compatible = "arm,cortex-a57", "arm,armv8";
161 reg = <0x20202>;
162 enable-method = "psci";
163 };
164
165 cpu11: cpu@20203 {
166 device_type = "cpu";
167 compatible = "arm,cortex-a57", "arm,armv8";
168 reg = <0x20203>;
169 enable-method = "psci";
170 };
171
172 cpu12: cpu@20300 {
173 device_type = "cpu";
174 compatible = "arm,cortex-a57", "arm,armv8";
175 reg = <0x20300>;
176 enable-method = "psci";
177 };
178
179 cpu13: cpu@20301 {
180 device_type = "cpu";
181 compatible = "arm,cortex-a57", "arm,armv8";
182 reg = <0x20301>;
183 enable-method = "psci";
184 };
185
186 cpu14: cpu@20302 {
187 device_type = "cpu";
188 compatible = "arm,cortex-a57", "arm,armv8";
189 reg = <0x20302>;
190 enable-method = "psci";
191 };
192
193 cpu15: cpu@20303 {
194 device_type = "cpu";
195 compatible = "arm,cortex-a57", "arm,armv8";
196 reg = <0x20303>;
197 enable-method = "psci";
198 };
199 };
200
201 gic: interrupt-controller@8d000000 {
202 compatible = "arm,gic-v3";
203 #interrupt-cells = <3>;
204 #address-cells = <2>;
205 #size-cells = <2>;
206 ranges;
207 interrupt-controller;
208 #redistributor-regions = <1>;
209 redistributor-stride = <0x0 0x30000>;
210 reg = <0x0 0x8d000000 0 0x10000>, /* GICD */
211 <0x0 0x8d100000 0 0x300000>, /* GICR */
212 <0x0 0xfe000000 0 0x10000>, /* GICC */
213 <0x0 0xfe010000 0 0x10000>, /* GICH */
214 <0x0 0xfe020000 0 0x10000>; /* GICV */
215 interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
216
217 its_totems: interrupt-controller@8c000000 {
218 compatible = "arm,gic-v3-its";
219 msi-controller;
220 reg = <0x0 0x8c000000 0x0 0x40000>;
221 };
222 };
223
224 timer {
225 compatible = "arm,armv8-timer";
226 interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
227 <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
228 <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
229 <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
230 };
231
232 pmu {
233 compatible = "arm,armv8-pmuv3";
234 interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
235 };
236
237 soc {
238 compatible = "simple-bus";
239 #address-cells = <2>;
240 #size-cells = <2>;
241 ranges;
242
243 refclk200mhz: refclk200mhz {
244 compatible = "fixed-clock";
245 #clock-cells = <0>;
246 clock-frequency = <200000000>;
247 };
248
249 uart0: uart@80300000 {
250 compatible = "snps,dw-apb-uart";
251 reg = <0x0 0x80300000 0x0 0x10000>;
252 interrupts = <GIC_SPI 317 IRQ_TYPE_LEVEL_HIGH>;
253 clocks = <&refclk200mhz>;
254 clock-names = "apb_pclk";
255 reg-shift = <2>;
256 reg-io-width = <4>;
257 status = "disabled";
258 };
259
260 uart1: uart@80310000 {
261 compatible = "snps,dw-apb-uart";
262 reg = <0x0 0x80310000 0x0 0x10000>;
263 interrupts = <GIC_SPI 318 IRQ_TYPE_LEVEL_HIGH>;
264 clocks = <&refclk200mhz>;
265 clock-names = "apb_pclk";
266 reg-shift = <2>;
267 reg-io-width = <4>;
268 status = "disabled";
269 };
270 };
271};
diff --git a/arch/arm64/boot/dts/hisilicon/hip05_hns.dtsi b/arch/arm64/boot/dts/hisilicon/hip05_hns.dtsi
new file mode 100644
index 000000000000..606dd5a05c2d
--- /dev/null
+++ b/arch/arm64/boot/dts/hisilicon/hip05_hns.dtsi
@@ -0,0 +1,191 @@
1soc0: soc@000000000 {
2 #address-cells = <2>;
3 #size-cells = <2>;
4 device_type = "soc";
5 compatible = "simple-bus";
6 ranges = <0x0 0x0 0x0 0x0 0x1 0x0>;
7 chip-id = <0>;
8
9 soc0_mdio0: mdio@803c0000 {
10 #address-cells = <1>;
11 #size-cells = <0>;
12 compatible = "hisilicon,hns-mdio";
13 reg = <0x0 0x803c0000 0x0 0x10000
14 0x0 0x80000000 0x0 0x10000>;
15
16 soc0_phy0: ethernet-phy@0 {
17 reg = <0x0>;
18 compatible = "ethernet-phy-ieee802.3-c22";
19 };
20 soc0_phy1: ethernet-phy@1 {
21 reg = <0x1>;
22 compatible = "ethernet-phy-ieee802.3-c22";
23 };
24 };
25
26 dsa: dsa@c7000000 {
27 compatible = "hisilicon,hns-dsaf-v1";
28 dsa_name = "dsaf0";
29 mode = "6port-16rss";
30 interrupt-parent = <&mbigen_dsa>;
31
32 reg = <0x0 0xC0000000 0x0 0x420000
33 0x0 0xC2000000 0x0 0x300000
34 0x0 0xc5000000 0x0 0x890000
35 0x0 0xc7000000 0x0 0x60000
36 >;
37
38 phy-handle = <0 0 0 0 &soc0_phy0 &soc0_phy1 0 0>;
39 interrupts = <
40 /* [14] ge fifo err 8 / xge 6**/
41 149 0x4 150 0x4 151 0x4 152 0x4
42 153 0x4 154 0x4 26 0x4 27 0x4
43 155 0x4 156 0x4 157 0x4 158 0x4 159 0x4 160 0x4
44 /* [12] rcb com 4*3**/
45 0x6 0x4 0x7 0x4 0x8 0x4 0x9 0x4
46 16 0x4 17 0x4 18 0x4 19 0x4
47 22 0x4 23 0x4 24 0x4 25 0x4
48 /* [8] ppe tnl 0-7***/
49 0x0 0x4 0x1 0x4 0x2 0x4 0x3 0x4
50 0x4 0x4 0x5 0x4 12 0x4 13 0x4
51 /* [21] dsaf event int 3+18**/
52 128 0x4 129 0x4 130 0x4
53 0x83 0x4 0x84 0x4 0x85 0x4 0x86 0x4 0x87 0x4 0x88 0x4
54 0x89 0x4 0x8a 0x4 0x8b 0x4 0x8c 0x4 0x8d 0x4 0x8e 0x4
55 0x8f 0x4 0x90 0x4 0x91 0x4 0x92 0x4 0x93 0x4 0x94 0x4
56 /* [4] debug rcb 2*2*/
57 0xe 0x1 0xf 0x1 0x14 0x1 0x15 0x1
58 /* [256] sevice rcb 2*128*/
59 0x180 0x1 0x181 0x1 0x182 0x1 0x183 0x1
60 0x184 0x1 0x185 0x1 0x186 0x1 0x187 0x1
61 0x188 0x1 0x189 0x1 0x18a 0x1 0x18b 0x1
62 0x18c 0x1 0x18d 0x1 0x18e 0x1 0x18f 0x1
63 0x190 0x1 0x191 0x1 0x192 0x1 0x193 0x1
64 0x194 0x1 0x195 0x1 0x196 0x1 0x197 0x1
65 0x198 0x1 0x199 0x1 0x19a 0x1 0x19b 0x1
66 0x19c 0x1 0x19d 0x1 0x19e 0x1 0x19f 0x1
67 0x1a0 0x1 0x1a1 0x1 0x1a2 0x1 0x1a3 0x1
68 0x1a4 0x1 0x1a5 0x1 0x1a6 0x1 0x1a7 0x1
69 0x1a8 0x1 0x1a9 0x1 0x1aa 0x1 0x1ab 0x1
70 0x1ac 0x1 0x1ad 0x1 0x1ae 0x1 0x1af 0x1
71 0x1b0 0x1 0x1b1 0x1 0x1b2 0x1 0x1b3 0x1
72 0x1b4 0x1 0x1b5 0x1 0x1b6 0x1 0x1b7 0x1
73 0x1b8 0x1 0x1b9 0x1 0x1ba 0x1 0x1bb 0x1
74 0x1bc 0x1 0x1bd 0x1 0x1be 0x1 0x1bf 0x1
75 0x1c0 0x1 0x1c1 0x1 0x1c2 0x1 0x1c3 0x1
76 0x1c4 0x1 0x1c5 0x1 0x1c6 0x1 0x1c7 0x1
77 0x1c8 0x1 0x1c9 0x1 0x1ca 0x1 0x1cb 0x1
78 0x1cc 0x1 0x1cd 0x1 0x1ce 0x1 0x1cf 0x1
79 0x1d0 0x1 0x1d1 0x1 0x1d2 0x1 0x1d3 0x1
80 0x1d4 0x1 0x1d5 0x1 0x1d6 0x1 0x1d7 0x1
81 0x1d8 0x1 0x1d9 0x1 0x1da 0x1 0x1db 0x1
82 0x1dc 0x1 0x1dd 0x1 0x1de 0x1 0x1df 0x1
83 0x1e0 0x1 0x1e1 0x1 0x1e2 0x1 0x1e3 0x1
84 0x1e4 0x1 0x1e5 0x1 0x1e6 0x1 0x1e7 0x1
85 0x1e8 0x1 0x1e9 0x1 0x1ea 0x1 0x1eb 0x1
86 0x1ec 0x1 0x1ed 0x1 0x1ee 0x1 0x1ef 0x1
87 0x1f0 0x1 0x1f1 0x1 0x1f2 0x1 0x1f3 0x1
88 0x1f4 0x1 0x1f5 0x1 0x1f6 0x1 0x1f7 0x1
89 0x1f8 0x1 0x1f9 0x1 0x1fa 0x1 0x1fb 0x1
90 0x1fc 0x1 0x1fd 0x1 0x1fe 0x1 0x1ff 0x1
91 0x200 0x1 0x201 0x1 0x202 0x1 0x203 0x1
92 0x204 0x1 0x205 0x1 0x206 0x1 0x207 0x1
93 0x208 0x1 0x209 0x1 0x20a 0x1 0x20b 0x1
94 0x20c 0x1 0x20d 0x1 0x20e 0x1 0x20f 0x1
95 0x210 0x1 0x211 0x1 0x212 0x1 0x213 0x1
96 0x214 0x1 0x215 0x1 0x216 0x1 0x217 0x1
97 0x218 0x1 0x219 0x1 0x21a 0x1 0x21b 0x1
98 0x21c 0x1 0x21d 0x1 0x21e 0x1 0x21f 0x1
99 0x220 0x1 0x221 0x1 0x222 0x1 0x223 0x1
100 0x224 0x1 0x225 0x1 0x226 0x1 0x227 0x1
101 0x228 0x1 0x229 0x1 0x22a 0x1 0x22b 0x1
102 0x22c 0x1 0x22d 0x1 0x22e 0x1 0x22f 0x1
103 0x230 0x1 0x231 0x1 0x232 0x1 0x233 0x1
104 0x234 0x1 0x235 0x1 0x236 0x1 0x237 0x1
105 0x238 0x1 0x239 0x1 0x23a 0x1 0x23b 0x1
106 0x23c 0x1 0x23d 0x1 0x23e 0x1 0x23f 0x1
107 0x240 0x1 0x241 0x1 0x242 0x1 0x243 0x1
108 0x244 0x1 0x245 0x1 0x246 0x1 0x247 0x1
109 0x248 0x1 0x249 0x1 0x24a 0x1 0x24b 0x1
110 0x24c 0x1 0x24d 0x1 0x24e 0x1 0x24f 0x1
111 0x250 0x1 0x251 0x1 0x252 0x1 0x253 0x1
112 0x254 0x1 0x255 0x1 0x256 0x1 0x257 0x1
113 0x258 0x1 0x259 0x1 0x25a 0x1 0x25b 0x1
114 0x25c 0x1 0x25d 0x1 0x25e 0x1 0x25f 0x1
115 0x260 0x1 0x261 0x1 0x262 0x1 0x263 0x1
116 0x264 0x1 0x265 0x1 0x266 0x1 0x267 0x1
117 0x268 0x1 0x269 0x1 0x26a 0x1 0x26b 0x1
118 0x26c 0x1 0x26d 0x1 0x26e 0x1 0x26f 0x1
119 0x270 0x1 0x271 0x1 0x272 0x1 0x273 0x1
120 0x274 0x1 0x275 0x1 0x276 0x1 0x277 0x1
121 0x278 0x1 0x279 0x1 0x27a 0x1 0x27b 0x1
122 0x27c 0x1 0x27d 0x1 0x27e 0x1 0x27f 0x1>;
123 buf-size = <4096>;
124 desc-num = <1024>;
125 dma-coherent;
126 };
127
128 eth0: ethernet@0{
129 compatible = "hisilicon,hns-nic-v1";
130 ae-name = "dsaf0";
131 port-id = <0>;
132 local-mac-address = [00 00 00 01 00 58];
133 status = "disabled";
134 dma-coherent;
135 };
136 eth1: ethernet@1{
137 compatible = "hisilicon,hns-nic-v1";
138 ae-name = "dsaf0";
139 port-id = <1>;
140 status = "disabled";
141 dma-coherent;
142 };
143 eth2: ethernet@2{
144 compatible = "hisilicon,hns-nic-v1";
145 ae-name = "dsaf0";
146 port-id = <2>;
147 local-mac-address = [00 00 00 01 00 5a];
148 status = "disabled";
149 dma-coherent;
150 };
151 eth3: ethernet@3{
152 compatible = "hisilicon,hns-nic-v1";
153 ae-name = "dsaf0";
154 port-id = <3>;
155 local-mac-address = [00 00 00 01 00 5b];
156 status = "disabled";
157 dma-coherent;
158 };
159 eth4: ethernet@4{
160 compatible = "hisilicon,hns-nic-v1";
161 ae-name = "dsaf0";
162 port-id = <4>;
163 local-mac-address = [00 00 00 01 00 5c];
164 status = "disabled";
165 dma-coherent;
166 };
167 eth5: ethernet@5{
168 compatible = "hisilicon,hns-nic-v1";
169 ae-name = "dsaf0";
170 port-id = <5>;
171 local-mac-address = [00 00 00 01 00 5d];
172 status = "disabled";
173 dma-coherent;
174 };
175 eth6: ethernet@6{
176 compatible = "hisilicon,hns-nic-v1";
177 ae-name = "dsaf0";
178 port-id = <6>;
179 local-mac-address = [00 00 00 01 00 5e];
180 status = "disabled";
181 dma-coherent;
182 };
183 eth7: ethernet@7{
184 compatible = "hisilicon,hns-nic-v1";
185 ae-name = "dsaf0";
186 port-id = <7>;
187 local-mac-address = [00 00 00 01 00 5f];
188 status = "disabled";
189 dma-coherent;
190 };
191};
diff --git a/arch/arm64/boot/dts/marvell/Makefile b/arch/arm64/boot/dts/marvell/Makefile
index e2f6afa7f849..348f4db4f313 100644
--- a/arch/arm64/boot/dts/marvell/Makefile
+++ b/arch/arm64/boot/dts/marvell/Makefile
@@ -1,4 +1,5 @@
1dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-dmp.dtb 1dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-dmp.dtb
2dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-stb.dtb
2 3
3always := $(dtb-y) 4always := $(dtb-y)
4subdir-y := $(dts-dirs) 5subdir-y := $(dts-dirs)
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts b/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
new file mode 100644
index 000000000000..348c37ecf069
--- /dev/null
+++ b/arch/arm64/boot/dts/marvell/berlin4ct-stb.dts
@@ -0,0 +1,66 @@
1/*
2 * Copyright (C) 2015 Marvell Technology Group Ltd.
3 *
4 * Author: Jisheng Zhang <jszhang@marvell.com>
5 *
6 * This file is dual-licensed: you can use it either under the terms
7 * of the GPLv2 or the X11 license, at your option. Note that this dual
8 * licensing only applies to this file, and not this project as a
9 * whole.
10 *
11 * a) This library is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License as
13 * published by the Free Software Foundation; either version 2 of the
14 * License, or (at your option) any later version.
15 *
16 * This library is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * Or, alternatively,
22 *
23 * b) Permission is hereby granted, free of charge, to any person
24 * obtaining a copy of this software and associated documentation
25 * files (the "Software"), to deal in the Software without
26 * restriction, including without limitation the rights to use,
27 * copy, modify, merge, publish, distribute, sublicense, and/or
28 * sell copies of the Software, and to permit persons to whom the
29 * Software is furnished to do so, subject to the following
30 * conditions:
31 *
32 * The above copyright notice and this permission notice shall be
33 * included in all copies or substantial portions of the Software.
34 *
35 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
36 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
37 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
38 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
39 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
40 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
41 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
42 * OTHER DEALINGS IN THE SOFTWARE.
43 */
44
45/dts-v1/;
46
47#include "berlin4ct.dtsi"
48
49/ {
50 model = "Marvell BG4CT STB board";
51 compatible = "marvell,berlin4ct-stb", "marvell,berlin4ct", "marvell,berlin";
52
53 chosen {
54 stdout-path = "serial0:115200n8";
55 };
56
57 memory {
58 device_type = "memory";
59 /* the first 16MB is for firmwares' usage */
60 reg = <0 0x01000000 0 0x7f000000>;
61 };
62};
63
64&uart0 {
65 status = "okay";
66};
diff --git a/arch/arm64/boot/dts/marvell/berlin4ct.dtsi b/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
index dd4a10d605d9..a3b5f1d4a240 100644
--- a/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
+++ b/arch/arm64/boot/dts/marvell/berlin4ct.dtsi
@@ -135,6 +135,96 @@
135 interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>; 135 interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
136 }; 136 };
137 137
138 apb@e80000 {
139 compatible = "simple-bus";
140 #address-cells = <1>;
141 #size-cells = <1>;
142
143 ranges = <0 0xe80000 0x10000>;
144 interrupt-parent = <&aic>;
145
146 gpio0: gpio@0400 {
147 compatible = "snps,dw-apb-gpio";
148 reg = <0x0400 0x400>;
149 #address-cells = <1>;
150 #size-cells = <0>;
151
152 porta: gpio-port@0 {
153 compatible = "snps,dw-apb-gpio-port";
154 gpio-controller;
155 #gpio-cells = <2>;
156 snps,nr-gpios = <32>;
157 reg = <0>;
158 interrupt-controller;
159 #interrupt-cells = <2>;
160 interrupts = <0>;
161 };
162 };
163
164 gpio1: gpio@0800 {
165 compatible = "snps,dw-apb-gpio";
166 reg = <0x0800 0x400>;
167 #address-cells = <1>;
168 #size-cells = <0>;
169
170 portb: gpio-port@1 {
171 compatible = "snps,dw-apb-gpio-port";
172 gpio-controller;
173 #gpio-cells = <2>;
174 snps,nr-gpios = <32>;
175 reg = <0>;
176 interrupt-controller;
177 #interrupt-cells = <2>;
178 interrupts = <1>;
179 };
180 };
181
182 gpio2: gpio@0c00 {
183 compatible = "snps,dw-apb-gpio";
184 reg = <0x0c00 0x400>;
185 #address-cells = <1>;
186 #size-cells = <0>;
187
188 portc: gpio-port@2 {
189 compatible = "snps,dw-apb-gpio-port";
190 gpio-controller;
191 #gpio-cells = <2>;
192 snps,nr-gpios = <32>;
193 reg = <0>;
194 interrupt-controller;
195 #interrupt-cells = <2>;
196 interrupts = <2>;
197 };
198 };
199
200 gpio3: gpio@1000 {
201 compatible = "snps,dw-apb-gpio";
202 reg = <0x1000 0x400>;
203 #address-cells = <1>;
204 #size-cells = <0>;
205
206 portd: gpio-port@3 {
207 compatible = "snps,dw-apb-gpio-port";
208 gpio-controller;
209 #gpio-cells = <2>;
210 snps,nr-gpios = <32>;
211 reg = <0>;
212 interrupt-controller;
213 #interrupt-cells = <2>;
214 interrupts = <3>;
215 };
216 };
217
218 aic: interrupt-controller@3800 {
219 compatible = "snps,dw-apb-ictl";
220 reg = <0x3800 0x30>;
221 interrupt-controller;
222 #interrupt-cells = <1>;
223 interrupt-parent = <&gic>;
224 interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
225 };
226 };
227
138 apb@fc0000 { 228 apb@fc0000 {
139 compatible = "simple-bus"; 229 compatible = "simple-bus";
140 #address-cells = <1>; 230 #address-cells = <1>;
@@ -151,6 +241,36 @@
151 interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>; 241 interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
152 }; 242 };
153 243
244 sm_gpio0: gpio@8000 {
245 compatible = "snps,dw-apb-gpio";
246 reg = <0x8000 0x400>;
247 #address-cells = <1>;
248 #size-cells = <0>;
249
250 porte: gpio-port@4 {
251 compatible = "snps,dw-apb-gpio-port";
252 gpio-controller;
253 #gpio-cells = <2>;
254 snps,nr-gpios = <32>;
255 reg = <0>;
256 };
257 };
258
259 sm_gpio1: gpio@9000 {
260 compatible = "snps,dw-apb-gpio";
261 reg = <0x9000 0x400>;
262 #address-cells = <1>;
263 #size-cells = <0>;
264
265 portf: gpio-port@5 {
266 compatible = "snps,dw-apb-gpio-port";
267 gpio-controller;
268 #gpio-cells = <2>;
269 snps,nr-gpios = <32>;
270 reg = <0>;
271 };
272 };
273
154 uart0: uart@d000 { 274 uart0: uart@d000 {
155 compatible = "snps,dw-apb-uart"; 275 compatible = "snps,dw-apb-uart";
156 reg = <0xd000 0x100>; 276 reg = <0xd000 0x100>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
index 4be66cadbc7c..811cb760ba49 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
@@ -387,6 +387,24 @@
387 }; 387 };
388}; 388};
389 389
390&pio {
391 spi_pins_a: spi0 {
392 pins_spi {
393 pinmux = <MT8173_PIN_69_SPI_CK__FUNC_SPI_CK_0_>,
394 <MT8173_PIN_70_SPI_MI__FUNC_SPI_MI_0_>,
395 <MT8173_PIN_71_SPI_MO__FUNC_SPI_MO_0_>,
396 <MT8173_PIN_72_SPI_CS__FUNC_SPI_CS_0_>;
397 };
398 };
399};
400
401&spi {
402 pinctrl-names = "default";
403 pinctrl-0 = <&spi_pins_a>;
404 mediatek,pad-select = <0>;
405 status = "okay";
406};
407
390&uart0 { 408&uart0 {
391 status = "okay"; 409 status = "okay";
392}; 410};
diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
index 06a15644be38..4dd5f93d0303 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
@@ -116,6 +116,13 @@
116 clock-output-names = "clk32k"; 116 clock-output-names = "clk32k";
117 }; 117 };
118 118
119 cpum_ck: oscillator@2 {
120 compatible = "fixed-clock";
121 #clock-cells = <0>;
122 clock-frequency = <0>;
123 clock-output-names = "cpum_ck";
124 };
125
119 timer { 126 timer {
120 compatible = "arm,armv8-timer"; 127 compatible = "arm,armv8-timer";
121 interrupt-parent = <&gic>; 128 interrupt-parent = <&gic>;
@@ -227,8 +234,10 @@
227 #power-domain-cells = <1>; 234 #power-domain-cells = <1>;
228 reg = <0 0x10006000 0 0x1000>; 235 reg = <0 0x10006000 0 0x1000>;
229 clocks = <&clk26m>, 236 clocks = <&clk26m>,
230 <&topckgen CLK_TOP_MM_SEL>; 237 <&topckgen CLK_TOP_MM_SEL>,
231 clock-names = "mfg", "mm"; 238 <&topckgen CLK_TOP_VENC_SEL>,
239 <&topckgen CLK_TOP_VENC_LT_SEL>;
240 clock-names = "mfg", "mm", "venc", "venc_lt";
232 infracfg = <&infracfg>; 241 infracfg = <&infracfg>;
233 }; 242 };
234 243
@@ -365,7 +374,20 @@
365 status = "disabled"; 374 status = "disabled";
366 }; 375 };
367 376
368 i2c3: i2c3@11010000 { 377 spi: spi@1100a000 {
378 compatible = "mediatek,mt8173-spi";
379 #address-cells = <1>;
380 #size-cells = <0>;
381 reg = <0 0x1100a000 0 0x1000>;
382 interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_LOW>;
383 clocks = <&topckgen CLK_TOP_SYSPLL3_D2>,
384 <&topckgen CLK_TOP_SPI_SEL>,
385 <&pericfg CLK_PERI_SPI0>;
386 clock-names = "parent-clk", "sel-clk", "spi-clk";
387 status = "disabled";
388 };
389
390 i2c3: i2c@11010000 {
369 compatible = "mediatek,mt8173-i2c"; 391 compatible = "mediatek,mt8173-i2c";
370 reg = <0 0x11010000 0 0x70>, 392 reg = <0 0x11010000 0 0x70>,
371 <0 0x11000280 0 0x80>; 393 <0 0x11000280 0 0x80>;
@@ -381,7 +403,7 @@
381 status = "disabled"; 403 status = "disabled";
382 }; 404 };
383 405
384 i2c4: i2c4@11011000 { 406 i2c4: i2c@11011000 {
385 compatible = "mediatek,mt8173-i2c"; 407 compatible = "mediatek,mt8173-i2c";
386 reg = <0 0x11011000 0 0x70>, 408 reg = <0 0x11011000 0 0x70>,
387 <0 0x11000300 0 0x80>; 409 <0 0x11000300 0 0x80>;
@@ -397,7 +419,7 @@
397 status = "disabled"; 419 status = "disabled";
398 }; 420 };
399 421
400 i2c6: i2c6@11013000 { 422 i2c6: i2c@11013000 {
401 compatible = "mediatek,mt8173-i2c"; 423 compatible = "mediatek,mt8173-i2c";
402 reg = <0 0x11013000 0 0x70>, 424 reg = <0 0x11013000 0 0x70>,
403 <0 0x11000080 0 0x80>; 425 <0 0x11000080 0 0x80>;
@@ -487,6 +509,36 @@
487 clock-names = "source", "hclk"; 509 clock-names = "source", "hclk";
488 status = "disabled"; 510 status = "disabled";
489 }; 511 };
512
513 mmsys: clock-controller@14000000 {
514 compatible = "mediatek,mt8173-mmsys", "syscon";
515 reg = <0 0x14000000 0 0x1000>;
516 #clock-cells = <1>;
517 };
518
519 imgsys: clock-controller@15000000 {
520 compatible = "mediatek,mt8173-imgsys", "syscon";
521 reg = <0 0x15000000 0 0x1000>;
522 #clock-cells = <1>;
523 };
524
525 vdecsys: clock-controller@16000000 {
526 compatible = "mediatek,mt8173-vdecsys", "syscon";
527 reg = <0 0x16000000 0 0x1000>;
528 #clock-cells = <1>;
529 };
530
531 vencsys: clock-controller@18000000 {
532 compatible = "mediatek,mt8173-vencsys", "syscon";
533 reg = <0 0x18000000 0 0x1000>;
534 #clock-cells = <1>;
535 };
536
537 vencltsys: clock-controller@19000000 {
538 compatible = "mediatek,mt8173-vencltsys", "syscon";
539 reg = <0 0x19000000 0 0x1000>;
540 #clock-cells = <1>;
541 };
490 }; 542 };
491}; 543};
492 544
diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
index 66804ffbc6d2..6b8abbe68746 100644
--- a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
@@ -19,6 +19,7 @@
19/ { 19/ {
20 aliases { 20 aliases {
21 serial0 = &blsp1_uart2; 21 serial0 = &blsp1_uart2;
22 serial1 = &blsp1_uart1;
22 }; 23 };
23 24
24 chosen { 25 chosen {
@@ -33,6 +34,31 @@
33 pinctrl-1 = <&blsp1_uart2_sleep>; 34 pinctrl-1 = <&blsp1_uart2_sleep>;
34 }; 35 };
35 36
37 i2c@78b6000 {
38 /* On Low speed expansion */
39 status = "okay";
40 };
41
42 i2c@78b8000 {
43 /* On High speed expansion */
44 status = "okay";
45 };
46
47 i2c@78ba000 {
48 /* On Low speed expansion */
49 status = "okay";
50 };
51
52 spi@78b7000 {
53 /* On High speed expansion */
54 status = "okay";
55 };
56
57 spi@78b9000 {
58 /* On Low speed expansion */
59 status = "okay";
60 };
61
36 leds { 62 leds {
37 pinctrl-names = "default"; 63 pinctrl-names = "default";
38 pinctrl-0 = <&msmgpio_leds>, 64 pinctrl-0 = <&msmgpio_leds>,
@@ -85,3 +111,7 @@
85 }; 111 };
86 }; 112 };
87}; 113};
114
115&sdhc_1 {
116 status = "okay";
117};
diff --git a/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi b/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi
index 568956859088..49ec55a37614 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi
@@ -13,6 +13,30 @@
13 13
14&msmgpio { 14&msmgpio {
15 15
16 blsp1_uart1_default: blsp1_uart1_default {
17 pinmux {
18 function = "blsp_uart1";
19 pins = "gpio0", "gpio1";
20 };
21 pinconf {
22 pins = "gpio0", "gpio1";
23 drive-strength = <16>;
24 bias-disable;
25 };
26 };
27
28 blsp1_uart1_sleep: blsp1_uart1_sleep {
29 pinmux {
30 function = "gpio";
31 pins = "gpio0", "gpio1";
32 };
33 pinconf {
34 pins = "gpio0", "gpio1";
35 drive-strength = <2>;
36 bias-pull-down;
37 };
38 };
39
16 blsp1_uart2_default: blsp1_uart2_default { 40 blsp1_uart2_default: blsp1_uart2_default {
17 pinmux { 41 pinmux {
18 function = "blsp_uart2"; 42 function = "blsp_uart2";
@@ -27,7 +51,7 @@
27 51
28 blsp1_uart2_sleep: blsp1_uart2_sleep { 52 blsp1_uart2_sleep: blsp1_uart2_sleep {
29 pinmux { 53 pinmux {
30 function = "blsp_uart2"; 54 function = "gpio";
31 pins = "gpio4", "gpio5"; 55 pins = "gpio4", "gpio5";
32 }; 56 };
33 pinconf { 57 pinconf {
@@ -241,6 +265,30 @@
241 }; 265 };
242 }; 266 };
243 267
268 i2c2_default: i2c2_default {
269 pinmux {
270 function = "blsp_i2c2";
271 pins = "gpio6", "gpio7";
272 };
273 pinconf {
274 pins = "gpio6", "gpio7";
275 drive-strength = <2>;
276 bias-disable = <0>;
277 };
278 };
279
280 i2c2_sleep: i2c2_sleep {
281 pinmux {
282 function = "gpio";
283 pins = "gpio6", "gpio7";
284 };
285 pinconf {
286 pins = "gpio6", "gpio7";
287 drive-strength = <2>;
288 bias-disable = <0>;
289 };
290 };
291
244 i2c4_default: i2c4_default { 292 i2c4_default: i2c4_default {
245 pinmux { 293 pinmux {
246 function = "blsp_i2c4"; 294 function = "blsp_i2c4";
@@ -255,7 +303,7 @@
255 303
256 i2c4_sleep: i2c4_sleep { 304 i2c4_sleep: i2c4_sleep {
257 pinmux { 305 pinmux {
258 function = "blsp_i2c4"; 306 function = "gpio";
259 pins = "gpio14", "gpio15"; 307 pins = "gpio14", "gpio15";
260 }; 308 };
261 pinconf { 309 pinconf {
@@ -265,6 +313,30 @@
265 }; 313 };
266 }; 314 };
267 315
316 i2c6_default: i2c6_default {
317 pinmux {
318 function = "blsp_i2c6";
319 pins = "gpio22", "gpio23";
320 };
321 pinconf {
322 pins = "gpio22", "gpio23";
323 drive-strength = <2>;
324 bias-disable = <0>;
325 };
326 };
327
328 i2c6_sleep: i2c6_sleep {
329 pinmux {
330 function = "gpio";
331 pins = "gpio22", "gpio23";
332 };
333 pinconf {
334 pins = "gpio22", "gpio23";
335 drive-strength = <2>;
336 bias-disable = <0>;
337 };
338 };
339
268 sdhc2_cd_pin { 340 sdhc2_cd_pin {
269 sdc2_cd_on: cd_on { 341 sdc2_cd_on: cd_on {
270 pinmux { 342 pinmux {
diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi
index 5911de008dd5..8d184ff19642 100644
--- a/arch/arm64/boot/dts/qcom/msm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi
@@ -99,9 +99,19 @@
99 compatible = "qcom,gcc-msm8916"; 99 compatible = "qcom,gcc-msm8916";
100 #clock-cells = <1>; 100 #clock-cells = <1>;
101 #reset-cells = <1>; 101 #reset-cells = <1>;
102 #power-domain-cells = <1>;
102 reg = <0x1800000 0x80000>; 103 reg = <0x1800000 0x80000>;
103 }; 104 };
104 105
106 blsp1_uart1: serial@78af000 {
107 compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
108 reg = <0x78af000 0x200>;
109 interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
110 clocks = <&gcc GCC_BLSP1_UART1_APPS_CLK>, <&gcc GCC_BLSP1_AHB_CLK>;
111 clock-names = "core", "iface";
112 status = "disabled";
113 };
114
105 blsp1_uart2: serial@78b0000 { 115 blsp1_uart2: serial@78b0000 {
106 compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm"; 116 compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
107 reg = <0x78b0000 0x200>; 117 reg = <0x78b0000 0x200>;
@@ -224,6 +234,21 @@
224 status = "disabled"; 234 status = "disabled";
225 }; 235 };
226 236
237 blsp_i2c2: i2c@78b6000 {
238 compatible = "qcom,i2c-qup-v2.2.1";
239 reg = <0x78b6000 0x1000>;
240 interrupts = <GIC_SPI 96 0>;
241 clocks = <&gcc GCC_BLSP1_AHB_CLK>,
242 <&gcc GCC_BLSP1_QUP2_I2C_APPS_CLK>;
243 clock-names = "iface", "core";
244 pinctrl-names = "default", "sleep";
245 pinctrl-0 = <&i2c2_default>;
246 pinctrl-1 = <&i2c2_sleep>;
247 #address-cells = <1>;
248 #size-cells = <0>;
249 status = "disabled";
250 };
251
227 blsp_i2c4: i2c@78b8000 { 252 blsp_i2c4: i2c@78b8000 {
228 compatible = "qcom,i2c-qup-v2.2.1"; 253 compatible = "qcom,i2c-qup-v2.2.1";
229 reg = <0x78b8000 0x1000>; 254 reg = <0x78b8000 0x1000>;
@@ -239,6 +264,21 @@
239 status = "disabled"; 264 status = "disabled";
240 }; 265 };
241 266
267 blsp_i2c6: i2c@78ba000 {
268 compatible = "qcom,i2c-qup-v2.2.1";
269 reg = <0x78ba000 0x1000>;
270 interrupts = <GIC_SPI 100 0>;
271 clocks = <&gcc GCC_BLSP1_AHB_CLK>,
272 <&gcc GCC_BLSP1_QUP6_I2C_APPS_CLK>;
273 clock-names = "iface", "core";
274 pinctrl-names = "default", "sleep";
275 pinctrl-0 = <&i2c6_default>;
276 pinctrl-1 = <&i2c6_sleep>;
277 #address-cells = <1>;
278 #size-cells = <0>;
279 status = "disabled";
280 };
281
242 sdhc_1: sdhci@07824000 { 282 sdhc_1: sdhci@07824000 {
243 compatible = "qcom,sdhci-msm-v4"; 283 compatible = "qcom,sdhci-msm-v4";
244 reg = <0x07824900 0x11c>, <0x07824000 0x800>; 284 reg = <0x07824900 0x11c>, <0x07824000 0x800>;
@@ -390,6 +430,13 @@
390 interrupt-controller; 430 interrupt-controller;
391 #interrupt-cells = <4>; 431 #interrupt-cells = <4>;
392 }; 432 };
433
434 rng@22000 {
435 compatible = "qcom,prng";
436 reg = <0x00022000 0x200>;
437 clocks = <&gcc GCC_PRNG_AHB_CLK>;
438 clock-names = "core";
439 };
393 }; 440 };
394}; 441};
395 442
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 34d71dd86781..bdd7aa358d2a 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -34,11 +34,12 @@ CONFIG_MODULE_UNLOAD=y
34CONFIG_ARCH_BCM_IPROC=y 34CONFIG_ARCH_BCM_IPROC=y
35CONFIG_ARCH_BERLIN=y 35CONFIG_ARCH_BERLIN=y
36CONFIG_ARCH_EXYNOS7=y 36CONFIG_ARCH_EXYNOS7=y
37CONFIG_ARCH_FSL_LS2085A=y 37CONFIG_ARCH_LAYERSCAPE=y
38CONFIG_ARCH_HISI=y 38CONFIG_ARCH_HISI=y
39CONFIG_ARCH_MEDIATEK=y 39CONFIG_ARCH_MEDIATEK=y
40CONFIG_ARCH_ROCKCHIP=y 40CONFIG_ARCH_ROCKCHIP=y
41CONFIG_ARCH_SEATTLE=y 41CONFIG_ARCH_SEATTLE=y
42CONFIG_ARCH_STRATIX10=y
42CONFIG_ARCH_TEGRA=y 43CONFIG_ARCH_TEGRA=y
43CONFIG_ARCH_TEGRA_132_SOC=y 44CONFIG_ARCH_TEGRA_132_SOC=y
44CONFIG_ARCH_QCOM=y 45CONFIG_ARCH_QCOM=y
@@ -49,8 +50,10 @@ CONFIG_ARCH_XGENE=y
49CONFIG_ARCH_ZYNQMP=y 50CONFIG_ARCH_ZYNQMP=y
50CONFIG_PCI=y 51CONFIG_PCI=y
51CONFIG_PCI_MSI=y 52CONFIG_PCI_MSI=y
53CONFIG_PCI_HOST_GENERIC=y
52CONFIG_PCI_XGENE=y 54CONFIG_PCI_XGENE=y
53CONFIG_SMP=y 55CONFIG_SMP=y
56CONFIG_SCHED_MC=y
54CONFIG_PREEMPT=y 57CONFIG_PREEMPT=y
55CONFIG_KSM=y 58CONFIG_KSM=y
56CONFIG_TRANSPARENT_HUGEPAGE=y 59CONFIG_TRANSPARENT_HUGEPAGE=y
@@ -109,6 +112,10 @@ CONFIG_SERIAL_8250_DW=y
109CONFIG_SERIAL_8250_MT6577=y 112CONFIG_SERIAL_8250_MT6577=y
110CONFIG_SERIAL_AMBA_PL011=y 113CONFIG_SERIAL_AMBA_PL011=y
111CONFIG_SERIAL_AMBA_PL011_CONSOLE=y 114CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
115CONFIG_SERIAL_SAMSUNG=y
116CONFIG_SERIAL_SAMSUNG_UARTS_4=y
117CONFIG_SERIAL_SAMSUNG_UARTS=4
118CONFIG_SERIAL_SAMSUNG_CONSOLE=y
112CONFIG_SERIAL_MSM=y 119CONFIG_SERIAL_MSM=y
113CONFIG_SERIAL_MSM_CONSOLE=y 120CONFIG_SERIAL_MSM_CONSOLE=y
114CONFIG_SERIAL_OF_PLATFORM=y 121CONFIG_SERIAL_OF_PLATFORM=y
@@ -116,8 +123,11 @@ CONFIG_SERIAL_XILINX_PS_UART=y
116CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y 123CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y
117CONFIG_VIRTIO_CONSOLE=y 124CONFIG_VIRTIO_CONSOLE=y
118# CONFIG_HW_RANDOM is not set 125# CONFIG_HW_RANDOM is not set
126CONFIG_I2C=y
127CONFIG_I2C_QUP=y
119CONFIG_SPI=y 128CONFIG_SPI=y
120CONFIG_SPI_PL022=y 129CONFIG_SPI_PL022=y
130CONFIG_SPI_QUP=y
121CONFIG_PINCTRL_MSM8916=y 131CONFIG_PINCTRL_MSM8916=y
122CONFIG_GPIO_PL061=y 132CONFIG_GPIO_PL061=y
123CONFIG_GPIO_XGENE=y 133CONFIG_GPIO_XGENE=y
@@ -126,6 +136,7 @@ CONFIG_POWER_RESET_SYSCON=y
126# CONFIG_HWMON is not set 136# CONFIG_HWMON is not set
127CONFIG_REGULATOR=y 137CONFIG_REGULATOR=y
128CONFIG_REGULATOR_FIXED_VOLTAGE=y 138CONFIG_REGULATOR_FIXED_VOLTAGE=y
139CONFIG_REGULATOR_QCOM_SMD_RPM=y
129CONFIG_FB=y 140CONFIG_FB=y
130CONFIG_FB_ARMCLCD=y 141CONFIG_FB_ARMCLCD=y
131CONFIG_FRAMEBUFFER_CONSOLE=y 142CONFIG_FRAMEBUFFER_CONSOLE=y
@@ -145,6 +156,10 @@ CONFIG_MMC_ARMMMCI=y
145CONFIG_MMC_SDHCI=y 156CONFIG_MMC_SDHCI=y
146CONFIG_MMC_SDHCI_PLTFM=y 157CONFIG_MMC_SDHCI_PLTFM=y
147CONFIG_MMC_SPI=y 158CONFIG_MMC_SPI=y
159CONFIG_MMC_DW=y
160CONFIG_MMC_DW_IDMAC=y
161CONFIG_MMC_DW_PLTFM=y
162CONFIG_MMC_DW_EXYNOS=y
148CONFIG_NEW_LEDS=y 163CONFIG_NEW_LEDS=y
149CONFIG_LEDS_CLASS=y 164CONFIG_LEDS_CLASS=y
150CONFIG_LEDS_SYSCON=y 165CONFIG_LEDS_SYSCON=y
@@ -154,12 +169,18 @@ CONFIG_LEDS_TRIGGER_CPU=y
154CONFIG_RTC_CLASS=y 169CONFIG_RTC_CLASS=y
155CONFIG_RTC_DRV_EFI=y 170CONFIG_RTC_DRV_EFI=y
156CONFIG_RTC_DRV_XGENE=y 171CONFIG_RTC_DRV_XGENE=y
172CONFIG_DMADEVICES=y
173CONFIG_QCOM_BAM_DMA=y
157CONFIG_VIRTIO_PCI=y 174CONFIG_VIRTIO_PCI=y
158CONFIG_VIRTIO_BALLOON=y 175CONFIG_VIRTIO_BALLOON=y
159CONFIG_VIRTIO_MMIO=y 176CONFIG_VIRTIO_MMIO=y
160CONFIG_COMMON_CLK_QCOM=y 177CONFIG_COMMON_CLK_QCOM=y
161CONFIG_MSM_GCC_8916=y 178CONFIG_MSM_GCC_8916=y
179CONFIG_HWSPINLOCK_QCOM=y
162# CONFIG_IOMMU_SUPPORT is not set 180# CONFIG_IOMMU_SUPPORT is not set
181CONFIG_QCOM_SMEM=y
182CONFIG_QCOM_SMD=y
183CONFIG_QCOM_SMD_RPM=y
163CONFIG_PHY_XGENE=y 184CONFIG_PHY_XGENE=y
164CONFIG_EXT2_FS=y 185CONFIG_EXT2_FS=y
165CONFIG_EXT3_FS=y 186CONFIG_EXT3_FS=y
@@ -203,3 +224,4 @@ CONFIG_CRYPTO_GHASH_ARM64_CE=y
203CONFIG_CRYPTO_AES_ARM64_CE_CCM=y 224CONFIG_CRYPTO_AES_ARM64_CE_CCM=y
204CONFIG_CRYPTO_AES_ARM64_CE_BLK=y 225CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
205CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y 226CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y
227CONFIG_CRYPTO_CRC32_ARM64=y
diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
index ce47792a983d..f7bd9bf0bbb3 100644
--- a/arch/arm64/crypto/aes-ce-cipher.c
+++ b/arch/arm64/crypto/aes-ce-cipher.c
@@ -237,7 +237,7 @@ EXPORT_SYMBOL(ce_aes_setkey);
237static struct crypto_alg aes_alg = { 237static struct crypto_alg aes_alg = {
238 .cra_name = "aes", 238 .cra_name = "aes",
239 .cra_driver_name = "aes-ce", 239 .cra_driver_name = "aes-ce",
240 .cra_priority = 300, 240 .cra_priority = 250,
241 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 241 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
242 .cra_blocksize = AES_BLOCK_SIZE, 242 .cra_blocksize = AES_BLOCK_SIZE,
243 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 243 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h
index 208cec08a74f..caafd63b8092 100644
--- a/arch/arm64/include/asm/acpi.h
+++ b/arch/arm64/include/asm/acpi.h
@@ -12,7 +12,6 @@
12#ifndef _ASM_ACPI_H 12#ifndef _ASM_ACPI_H
13#define _ASM_ACPI_H 13#define _ASM_ACPI_H
14 14
15#include <linux/irqchip/arm-gic-acpi.h>
16#include <linux/mm.h> 15#include <linux/mm.h>
17#include <linux/psci.h> 16#include <linux/psci.h>
18 17
@@ -92,4 +91,9 @@ static inline const char *acpi_get_enable_method(int cpu)
92{ 91{
93 return acpi_psci_present() ? "psci" : NULL; 92 return acpi_psci_present() ? "psci" : NULL;
94} 93}
94
95#ifdef CONFIG_ACPI_APEI
96pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr);
97#endif
98
95#endif /*_ASM_ACPI_H*/ 99#endif /*_ASM_ACPI_H*/
diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h
new file mode 100644
index 000000000000..030cdcb46c6b
--- /dev/null
+++ b/arch/arm64/include/asm/arch_gicv3.h
@@ -0,0 +1,170 @@
1/*
2 * arch/arm64/include/asm/arch_gicv3.h
3 *
4 * Copyright (C) 2015 ARM Ltd.
5 *
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18#ifndef __ASM_ARCH_GICV3_H
19#define __ASM_ARCH_GICV3_H
20
21#include <asm/sysreg.h>
22
23#define ICC_EOIR1_EL1 sys_reg(3, 0, 12, 12, 1)
24#define ICC_DIR_EL1 sys_reg(3, 0, 12, 11, 1)
25#define ICC_IAR1_EL1 sys_reg(3, 0, 12, 12, 0)
26#define ICC_SGI1R_EL1 sys_reg(3, 0, 12, 11, 5)
27#define ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
28#define ICC_CTLR_EL1 sys_reg(3, 0, 12, 12, 4)
29#define ICC_SRE_EL1 sys_reg(3, 0, 12, 12, 5)
30#define ICC_GRPEN1_EL1 sys_reg(3, 0, 12, 12, 7)
31
32#define ICC_SRE_EL2 sys_reg(3, 4, 12, 9, 5)
33
34/*
35 * System register definitions
36 */
37#define ICH_VSEIR_EL2 sys_reg(3, 4, 12, 9, 4)
38#define ICH_HCR_EL2 sys_reg(3, 4, 12, 11, 0)
39#define ICH_VTR_EL2 sys_reg(3, 4, 12, 11, 1)
40#define ICH_MISR_EL2 sys_reg(3, 4, 12, 11, 2)
41#define ICH_EISR_EL2 sys_reg(3, 4, 12, 11, 3)
42#define ICH_ELSR_EL2 sys_reg(3, 4, 12, 11, 5)
43#define ICH_VMCR_EL2 sys_reg(3, 4, 12, 11, 7)
44
45#define __LR0_EL2(x) sys_reg(3, 4, 12, 12, x)
46#define __LR8_EL2(x) sys_reg(3, 4, 12, 13, x)
47
48#define ICH_LR0_EL2 __LR0_EL2(0)
49#define ICH_LR1_EL2 __LR0_EL2(1)
50#define ICH_LR2_EL2 __LR0_EL2(2)
51#define ICH_LR3_EL2 __LR0_EL2(3)
52#define ICH_LR4_EL2 __LR0_EL2(4)
53#define ICH_LR5_EL2 __LR0_EL2(5)
54#define ICH_LR6_EL2 __LR0_EL2(6)
55#define ICH_LR7_EL2 __LR0_EL2(7)
56#define ICH_LR8_EL2 __LR8_EL2(0)
57#define ICH_LR9_EL2 __LR8_EL2(1)
58#define ICH_LR10_EL2 __LR8_EL2(2)
59#define ICH_LR11_EL2 __LR8_EL2(3)
60#define ICH_LR12_EL2 __LR8_EL2(4)
61#define ICH_LR13_EL2 __LR8_EL2(5)
62#define ICH_LR14_EL2 __LR8_EL2(6)
63#define ICH_LR15_EL2 __LR8_EL2(7)
64
65#define __AP0Rx_EL2(x) sys_reg(3, 4, 12, 8, x)
66#define ICH_AP0R0_EL2 __AP0Rx_EL2(0)
67#define ICH_AP0R1_EL2 __AP0Rx_EL2(1)
68#define ICH_AP0R2_EL2 __AP0Rx_EL2(2)
69#define ICH_AP0R3_EL2 __AP0Rx_EL2(3)
70
71#define __AP1Rx_EL2(x) sys_reg(3, 4, 12, 9, x)
72#define ICH_AP1R0_EL2 __AP1Rx_EL2(0)
73#define ICH_AP1R1_EL2 __AP1Rx_EL2(1)
74#define ICH_AP1R2_EL2 __AP1Rx_EL2(2)
75#define ICH_AP1R3_EL2 __AP1Rx_EL2(3)
76
77#ifndef __ASSEMBLY__
78
79#include <linux/stringify.h>
80
81/*
82 * Low-level accessors
83 *
84 * These system registers are 32 bits, but we make sure that the compiler
85 * sets the GP register's most significant bits to 0 with an explicit cast.
86 */
87
88static inline void gic_write_eoir(u32 irq)
89{
90 asm volatile("msr_s " __stringify(ICC_EOIR1_EL1) ", %0" : : "r" ((u64)irq));
91 isb();
92}
93
94static inline void gic_write_dir(u32 irq)
95{
96 asm volatile("msr_s " __stringify(ICC_DIR_EL1) ", %0" : : "r" ((u64)irq));
97 isb();
98}
99
100static inline u64 gic_read_iar_common(void)
101{
102 u64 irqstat;
103
104 asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat));
105 return irqstat;
106}
107
108/*
109 * Cavium ThunderX erratum 23154
110 *
111 * The gicv3 of ThunderX requires a modified version for reading the
112 * IAR status to ensure data synchronization (access to icc_iar1_el1
113 * is not sync'ed before and after).
114 */
115static inline u64 gic_read_iar_cavium_thunderx(void)
116{
117 u64 irqstat;
118
119 asm volatile(
120 "nop;nop;nop;nop\n\t"
121 "nop;nop;nop;nop\n\t"
122 "mrs_s %0, " __stringify(ICC_IAR1_EL1) "\n\t"
123 "nop;nop;nop;nop"
124 : "=r" (irqstat));
125 mb();
126
127 return irqstat;
128}
129
130static inline void gic_write_pmr(u32 val)
131{
132 asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" ((u64)val));
133}
134
135static inline void gic_write_ctlr(u32 val)
136{
137 asm volatile("msr_s " __stringify(ICC_CTLR_EL1) ", %0" : : "r" ((u64)val));
138 isb();
139}
140
141static inline void gic_write_grpen1(u32 val)
142{
143 asm volatile("msr_s " __stringify(ICC_GRPEN1_EL1) ", %0" : : "r" ((u64)val));
144 isb();
145}
146
147static inline void gic_write_sgi1r(u64 val)
148{
149 asm volatile("msr_s " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val));
150}
151
152static inline u32 gic_read_sre(void)
153{
154 u64 val;
155
156 asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
157 return val;
158}
159
160static inline void gic_write_sre(u32 val)
161{
162 asm volatile("msr_s " __stringify(ICC_SRE_EL1) ", %0" : : "r" ((u64)val));
163 isb();
164}
165
166#define gic_read_typer(c) readq_relaxed(c)
167#define gic_write_irouter(v, c) writeq_relaxed(v, c)
168
169#endif /* __ASSEMBLY__ */
170#endif /* __ASM_ARCH_GICV3_H */
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index b51f2cc22ca9..12eff928ef8b 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -193,4 +193,15 @@ lr .req x30 // link register
193 str \src, [\tmp, :lo12:\sym] 193 str \src, [\tmp, :lo12:\sym]
194 .endm 194 .endm
195 195
196/*
197 * Annotate a function as position independent, i.e., safe to be called before
198 * the kernel virtual mapping is activated.
199 */
200#define ENDPIPROC(x) \
201 .globl __pi_##x; \
202 .type __pi_##x, %function; \
203 .set __pi_##x, x; \
204 .size __pi_##x, . - x; \
205 ENDPROC(x)
206
196#endif /* __ASM_ASSEMBLER_H */ 207#endif /* __ASM_ASSEMBLER_H */
diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h
index 35a67783cfa0..f3a3586a421c 100644
--- a/arch/arm64/include/asm/atomic.h
+++ b/arch/arm64/include/asm/atomic.h
@@ -54,14 +54,43 @@
54#define ATOMIC_INIT(i) { (i) } 54#define ATOMIC_INIT(i) { (i) }
55 55
56#define atomic_read(v) READ_ONCE((v)->counter) 56#define atomic_read(v) READ_ONCE((v)->counter)
57#define atomic_set(v, i) (((v)->counter) = (i)) 57#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
58
59#define atomic_add_return_relaxed atomic_add_return_relaxed
60#define atomic_add_return_acquire atomic_add_return_acquire
61#define atomic_add_return_release atomic_add_return_release
62#define atomic_add_return atomic_add_return
63
64#define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v))
65#define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v))
66#define atomic_inc_return_release(v) atomic_add_return_release(1, (v))
67#define atomic_inc_return(v) atomic_add_return(1, (v))
68
69#define atomic_sub_return_relaxed atomic_sub_return_relaxed
70#define atomic_sub_return_acquire atomic_sub_return_acquire
71#define atomic_sub_return_release atomic_sub_return_release
72#define atomic_sub_return atomic_sub_return
73
74#define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v))
75#define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v))
76#define atomic_dec_return_release(v) atomic_sub_return_release(1, (v))
77#define atomic_dec_return(v) atomic_sub_return(1, (v))
78
79#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
80#define atomic_xchg_acquire(v, new) xchg_acquire(&((v)->counter), (new))
81#define atomic_xchg_release(v, new) xchg_release(&((v)->counter), (new))
58#define atomic_xchg(v, new) xchg(&((v)->counter), (new)) 82#define atomic_xchg(v, new) xchg(&((v)->counter), (new))
83
84#define atomic_cmpxchg_relaxed(v, old, new) \
85 cmpxchg_relaxed(&((v)->counter), (old), (new))
86#define atomic_cmpxchg_acquire(v, old, new) \
87 cmpxchg_acquire(&((v)->counter), (old), (new))
88#define atomic_cmpxchg_release(v, old, new) \
89 cmpxchg_release(&((v)->counter), (old), (new))
59#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new)) 90#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new))
60 91
61#define atomic_inc(v) atomic_add(1, (v)) 92#define atomic_inc(v) atomic_add(1, (v))
62#define atomic_dec(v) atomic_sub(1, (v)) 93#define atomic_dec(v) atomic_sub(1, (v))
63#define atomic_inc_return(v) atomic_add_return(1, (v))
64#define atomic_dec_return(v) atomic_sub_return(1, (v))
65#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) 94#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
66#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) 95#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0)
67#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) 96#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
@@ -75,13 +104,39 @@
75#define ATOMIC64_INIT ATOMIC_INIT 104#define ATOMIC64_INIT ATOMIC_INIT
76#define atomic64_read atomic_read 105#define atomic64_read atomic_read
77#define atomic64_set atomic_set 106#define atomic64_set atomic_set
107
108#define atomic64_add_return_relaxed atomic64_add_return_relaxed
109#define atomic64_add_return_acquire atomic64_add_return_acquire
110#define atomic64_add_return_release atomic64_add_return_release
111#define atomic64_add_return atomic64_add_return
112
113#define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v))
114#define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v))
115#define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v))
116#define atomic64_inc_return(v) atomic64_add_return(1, (v))
117
118#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
119#define atomic64_sub_return_acquire atomic64_sub_return_acquire
120#define atomic64_sub_return_release atomic64_sub_return_release
121#define atomic64_sub_return atomic64_sub_return
122
123#define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v))
124#define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v))
125#define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v))
126#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
127
128#define atomic64_xchg_relaxed atomic_xchg_relaxed
129#define atomic64_xchg_acquire atomic_xchg_acquire
130#define atomic64_xchg_release atomic_xchg_release
78#define atomic64_xchg atomic_xchg 131#define atomic64_xchg atomic_xchg
132
133#define atomic64_cmpxchg_relaxed atomic_cmpxchg_relaxed
134#define atomic64_cmpxchg_acquire atomic_cmpxchg_acquire
135#define atomic64_cmpxchg_release atomic_cmpxchg_release
79#define atomic64_cmpxchg atomic_cmpxchg 136#define atomic64_cmpxchg atomic_cmpxchg
80 137
81#define atomic64_inc(v) atomic64_add(1, (v)) 138#define atomic64_inc(v) atomic64_add(1, (v))
82#define atomic64_dec(v) atomic64_sub(1, (v)) 139#define atomic64_dec(v) atomic64_sub(1, (v))
83#define atomic64_inc_return(v) atomic64_add_return(1, (v))
84#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
85#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) 140#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
86#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0) 141#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0)
87#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0) 142#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
index b3b5c4ae3800..f61c84f6ba02 100644
--- a/arch/arm64/include/asm/atomic_ll_sc.h
+++ b/arch/arm64/include/asm/atomic_ll_sc.h
@@ -55,40 +55,47 @@ __LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \
55} \ 55} \
56__LL_SC_EXPORT(atomic_##op); 56__LL_SC_EXPORT(atomic_##op);
57 57
58#define ATOMIC_OP_RETURN(op, asm_op) \ 58#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
59__LL_SC_INLINE int \ 59__LL_SC_INLINE int \
60__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v)) \ 60__LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v)) \
61{ \ 61{ \
62 unsigned long tmp; \ 62 unsigned long tmp; \
63 int result; \ 63 int result; \
64 \ 64 \
65 asm volatile("// atomic_" #op "_return\n" \ 65 asm volatile("// atomic_" #op "_return" #name "\n" \
66" prfm pstl1strm, %2\n" \ 66" prfm pstl1strm, %2\n" \
67"1: ldxr %w0, %2\n" \ 67"1: ld" #acq "xr %w0, %2\n" \
68" " #asm_op " %w0, %w0, %w3\n" \ 68" " #asm_op " %w0, %w0, %w3\n" \
69" stlxr %w1, %w0, %2\n" \ 69" st" #rel "xr %w1, %w0, %2\n" \
70" cbnz %w1, 1b" \ 70" cbnz %w1, 1b\n" \
71" " #mb \
71 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 72 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
72 : "Ir" (i) \ 73 : "Ir" (i) \
73 : "memory"); \ 74 : cl); \
74 \ 75 \
75 smp_mb(); \
76 return result; \ 76 return result; \
77} \ 77} \
78__LL_SC_EXPORT(atomic_##op##_return); 78__LL_SC_EXPORT(atomic_##op##_return##name);
79 79
80#define ATOMIC_OPS(op, asm_op) \ 80#define ATOMIC_OPS(...) \
81 ATOMIC_OP(op, asm_op) \ 81 ATOMIC_OP(__VA_ARGS__) \
82 ATOMIC_OP_RETURN(op, asm_op) 82 ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)
83 83
84ATOMIC_OPS(add, add) 84#define ATOMIC_OPS_RLX(...) \
85ATOMIC_OPS(sub, sub) 85 ATOMIC_OPS(__VA_ARGS__) \
86 ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\
87 ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\
88 ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)
89
90ATOMIC_OPS_RLX(add, add)
91ATOMIC_OPS_RLX(sub, sub)
86 92
87ATOMIC_OP(and, and) 93ATOMIC_OP(and, and)
88ATOMIC_OP(andnot, bic) 94ATOMIC_OP(andnot, bic)
89ATOMIC_OP(or, orr) 95ATOMIC_OP(or, orr)
90ATOMIC_OP(xor, eor) 96ATOMIC_OP(xor, eor)
91 97
98#undef ATOMIC_OPS_RLX
92#undef ATOMIC_OPS 99#undef ATOMIC_OPS
93#undef ATOMIC_OP_RETURN 100#undef ATOMIC_OP_RETURN
94#undef ATOMIC_OP 101#undef ATOMIC_OP
@@ -111,40 +118,47 @@ __LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \
111} \ 118} \
112__LL_SC_EXPORT(atomic64_##op); 119__LL_SC_EXPORT(atomic64_##op);
113 120
114#define ATOMIC64_OP_RETURN(op, asm_op) \ 121#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
115__LL_SC_INLINE long \ 122__LL_SC_INLINE long \
116__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v)) \ 123__LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v)) \
117{ \ 124{ \
118 long result; \ 125 long result; \
119 unsigned long tmp; \ 126 unsigned long tmp; \
120 \ 127 \
121 asm volatile("// atomic64_" #op "_return\n" \ 128 asm volatile("// atomic64_" #op "_return" #name "\n" \
122" prfm pstl1strm, %2\n" \ 129" prfm pstl1strm, %2\n" \
123"1: ldxr %0, %2\n" \ 130"1: ld" #acq "xr %0, %2\n" \
124" " #asm_op " %0, %0, %3\n" \ 131" " #asm_op " %0, %0, %3\n" \
125" stlxr %w1, %0, %2\n" \ 132" st" #rel "xr %w1, %0, %2\n" \
126" cbnz %w1, 1b" \ 133" cbnz %w1, 1b\n" \
134" " #mb \
127 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 135 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
128 : "Ir" (i) \ 136 : "Ir" (i) \
129 : "memory"); \ 137 : cl); \
130 \ 138 \
131 smp_mb(); \
132 return result; \ 139 return result; \
133} \ 140} \
134__LL_SC_EXPORT(atomic64_##op##_return); 141__LL_SC_EXPORT(atomic64_##op##_return##name);
142
143#define ATOMIC64_OPS(...) \
144 ATOMIC64_OP(__VA_ARGS__) \
145 ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__)
135 146
136#define ATOMIC64_OPS(op, asm_op) \ 147#define ATOMIC64_OPS_RLX(...) \
137 ATOMIC64_OP(op, asm_op) \ 148 ATOMIC64_OPS(__VA_ARGS__) \
138 ATOMIC64_OP_RETURN(op, asm_op) 149 ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \
150 ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \
151 ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__)
139 152
140ATOMIC64_OPS(add, add) 153ATOMIC64_OPS_RLX(add, add)
141ATOMIC64_OPS(sub, sub) 154ATOMIC64_OPS_RLX(sub, sub)
142 155
143ATOMIC64_OP(and, and) 156ATOMIC64_OP(and, and)
144ATOMIC64_OP(andnot, bic) 157ATOMIC64_OP(andnot, bic)
145ATOMIC64_OP(or, orr) 158ATOMIC64_OP(or, orr)
146ATOMIC64_OP(xor, eor) 159ATOMIC64_OP(xor, eor)
147 160
161#undef ATOMIC64_OPS_RLX
148#undef ATOMIC64_OPS 162#undef ATOMIC64_OPS
149#undef ATOMIC64_OP_RETURN 163#undef ATOMIC64_OP_RETURN
150#undef ATOMIC64_OP 164#undef ATOMIC64_OP
@@ -172,7 +186,7 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
172} 186}
173__LL_SC_EXPORT(atomic64_dec_if_positive); 187__LL_SC_EXPORT(atomic64_dec_if_positive);
174 188
175#define __CMPXCHG_CASE(w, sz, name, mb, rel, cl) \ 189#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl) \
176__LL_SC_INLINE unsigned long \ 190__LL_SC_INLINE unsigned long \
177__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \ 191__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
178 unsigned long old, \ 192 unsigned long old, \
@@ -182,7 +196,7 @@ __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
182 \ 196 \
183 asm volatile( \ 197 asm volatile( \
184 " prfm pstl1strm, %[v]\n" \ 198 " prfm pstl1strm, %[v]\n" \
185 "1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \ 199 "1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \
186 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ 200 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
187 " cbnz %" #w "[tmp], 2f\n" \ 201 " cbnz %" #w "[tmp], 2f\n" \
188 " st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \ 202 " st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
@@ -199,19 +213,27 @@ __LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
199} \ 213} \
200__LL_SC_EXPORT(__cmpxchg_case_##name); 214__LL_SC_EXPORT(__cmpxchg_case_##name);
201 215
202__CMPXCHG_CASE(w, b, 1, , , ) 216__CMPXCHG_CASE(w, b, 1, , , , )
203__CMPXCHG_CASE(w, h, 2, , , ) 217__CMPXCHG_CASE(w, h, 2, , , , )
204__CMPXCHG_CASE(w, , 4, , , ) 218__CMPXCHG_CASE(w, , 4, , , , )
205__CMPXCHG_CASE( , , 8, , , ) 219__CMPXCHG_CASE( , , 8, , , , )
206__CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory") 220__CMPXCHG_CASE(w, b, acq_1, , a, , "memory")
207__CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory") 221__CMPXCHG_CASE(w, h, acq_2, , a, , "memory")
208__CMPXCHG_CASE(w, , mb_4, dmb ish, l, "memory") 222__CMPXCHG_CASE(w, , acq_4, , a, , "memory")
209__CMPXCHG_CASE( , , mb_8, dmb ish, l, "memory") 223__CMPXCHG_CASE( , , acq_8, , a, , "memory")
224__CMPXCHG_CASE(w, b, rel_1, , , l, "memory")
225__CMPXCHG_CASE(w, h, rel_2, , , l, "memory")
226__CMPXCHG_CASE(w, , rel_4, , , l, "memory")
227__CMPXCHG_CASE( , , rel_8, , , l, "memory")
228__CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory")
229__CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory")
230__CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory")
231__CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory")
210 232
211#undef __CMPXCHG_CASE 233#undef __CMPXCHG_CASE
212 234
213#define __CMPXCHG_DBL(name, mb, rel, cl) \ 235#define __CMPXCHG_DBL(name, mb, rel, cl) \
214__LL_SC_INLINE int \ 236__LL_SC_INLINE long \
215__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \ 237__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
216 unsigned long old2, \ 238 unsigned long old2, \
217 unsigned long new1, \ 239 unsigned long new1, \
diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h
index 55d740e63459..197e06afbf71 100644
--- a/arch/arm64/include/asm/atomic_lse.h
+++ b/arch/arm64/include/asm/atomic_lse.h
@@ -75,24 +75,32 @@ static inline void atomic_add(int i, atomic_t *v)
75 : "x30"); 75 : "x30");
76} 76}
77 77
78static inline int atomic_add_return(int i, atomic_t *v) 78#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
79{ 79static inline int atomic_add_return##name(int i, atomic_t *v) \
80 register int w0 asm ("w0") = i; 80{ \
81 register atomic_t *x1 asm ("x1") = v; 81 register int w0 asm ("w0") = i; \
82 register atomic_t *x1 asm ("x1") = v; \
83 \
84 asm volatile(ARM64_LSE_ATOMIC_INSN( \
85 /* LL/SC */ \
86 " nop\n" \
87 __LL_SC_ATOMIC(add_return##name), \
88 /* LSE atomics */ \
89 " ldadd" #mb " %w[i], w30, %[v]\n" \
90 " add %w[i], %w[i], w30") \
91 : [i] "+r" (w0), [v] "+Q" (v->counter) \
92 : "r" (x1) \
93 : "x30" , ##cl); \
94 \
95 return w0; \
96}
82 97
83 asm volatile(ARM64_LSE_ATOMIC_INSN( 98ATOMIC_OP_ADD_RETURN(_relaxed, )
84 /* LL/SC */ 99ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
85 " nop\n" 100ATOMIC_OP_ADD_RETURN(_release, l, "memory")
86 __LL_SC_ATOMIC(add_return), 101ATOMIC_OP_ADD_RETURN( , al, "memory")
87 /* LSE atomics */
88 " ldaddal %w[i], w30, %[v]\n"
89 " add %w[i], %w[i], w30")
90 : [i] "+r" (w0), [v] "+Q" (v->counter)
91 : "r" (x1)
92 : "x30", "memory");
93 102
94 return w0; 103#undef ATOMIC_OP_ADD_RETURN
95}
96 104
97static inline void atomic_and(int i, atomic_t *v) 105static inline void atomic_and(int i, atomic_t *v)
98{ 106{
@@ -128,27 +136,34 @@ static inline void atomic_sub(int i, atomic_t *v)
128 : "x30"); 136 : "x30");
129} 137}
130 138
131static inline int atomic_sub_return(int i, atomic_t *v) 139#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
132{ 140static inline int atomic_sub_return##name(int i, atomic_t *v) \
133 register int w0 asm ("w0") = i; 141{ \
134 register atomic_t *x1 asm ("x1") = v; 142 register int w0 asm ("w0") = i; \
135 143 register atomic_t *x1 asm ("x1") = v; \
136 asm volatile(ARM64_LSE_ATOMIC_INSN( 144 \
137 /* LL/SC */ 145 asm volatile(ARM64_LSE_ATOMIC_INSN( \
138 " nop\n" 146 /* LL/SC */ \
139 __LL_SC_ATOMIC(sub_return) 147 " nop\n" \
140 " nop", 148 __LL_SC_ATOMIC(sub_return##name) \
141 /* LSE atomics */ 149 " nop", \
142 " neg %w[i], %w[i]\n" 150 /* LSE atomics */ \
143 " ldaddal %w[i], w30, %[v]\n" 151 " neg %w[i], %w[i]\n" \
144 " add %w[i], %w[i], w30") 152 " ldadd" #mb " %w[i], w30, %[v]\n" \
145 : [i] "+r" (w0), [v] "+Q" (v->counter) 153 " add %w[i], %w[i], w30") \
146 : "r" (x1) 154 : [i] "+r" (w0), [v] "+Q" (v->counter) \
147 : "x30", "memory"); 155 : "r" (x1) \
148 156 : "x30" , ##cl); \
149 return w0; 157 \
158 return w0; \
150} 159}
151 160
161ATOMIC_OP_SUB_RETURN(_relaxed, )
162ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
163ATOMIC_OP_SUB_RETURN(_release, l, "memory")
164ATOMIC_OP_SUB_RETURN( , al, "memory")
165
166#undef ATOMIC_OP_SUB_RETURN
152#undef __LL_SC_ATOMIC 167#undef __LL_SC_ATOMIC
153 168
154#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) 169#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
@@ -201,24 +216,32 @@ static inline void atomic64_add(long i, atomic64_t *v)
201 : "x30"); 216 : "x30");
202} 217}
203 218
204static inline long atomic64_add_return(long i, atomic64_t *v) 219#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
205{ 220static inline long atomic64_add_return##name(long i, atomic64_t *v) \
206 register long x0 asm ("x0") = i; 221{ \
207 register atomic64_t *x1 asm ("x1") = v; 222 register long x0 asm ("x0") = i; \
223 register atomic64_t *x1 asm ("x1") = v; \
224 \
225 asm volatile(ARM64_LSE_ATOMIC_INSN( \
226 /* LL/SC */ \
227 " nop\n" \
228 __LL_SC_ATOMIC64(add_return##name), \
229 /* LSE atomics */ \
230 " ldadd" #mb " %[i], x30, %[v]\n" \
231 " add %[i], %[i], x30") \
232 : [i] "+r" (x0), [v] "+Q" (v->counter) \
233 : "r" (x1) \
234 : "x30" , ##cl); \
235 \
236 return x0; \
237}
208 238
209 asm volatile(ARM64_LSE_ATOMIC_INSN( 239ATOMIC64_OP_ADD_RETURN(_relaxed, )
210 /* LL/SC */ 240ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
211 " nop\n" 241ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
212 __LL_SC_ATOMIC64(add_return), 242ATOMIC64_OP_ADD_RETURN( , al, "memory")
213 /* LSE atomics */
214 " ldaddal %[i], x30, %[v]\n"
215 " add %[i], %[i], x30")
216 : [i] "+r" (x0), [v] "+Q" (v->counter)
217 : "r" (x1)
218 : "x30", "memory");
219 243
220 return x0; 244#undef ATOMIC64_OP_ADD_RETURN
221}
222 245
223static inline void atomic64_and(long i, atomic64_t *v) 246static inline void atomic64_and(long i, atomic64_t *v)
224{ 247{
@@ -254,26 +277,34 @@ static inline void atomic64_sub(long i, atomic64_t *v)
254 : "x30"); 277 : "x30");
255} 278}
256 279
257static inline long atomic64_sub_return(long i, atomic64_t *v) 280#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
258{ 281static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
259 register long x0 asm ("x0") = i; 282{ \
260 register atomic64_t *x1 asm ("x1") = v; 283 register long x0 asm ("x0") = i; \
284 register atomic64_t *x1 asm ("x1") = v; \
285 \
286 asm volatile(ARM64_LSE_ATOMIC_INSN( \
287 /* LL/SC */ \
288 " nop\n" \
289 __LL_SC_ATOMIC64(sub_return##name) \
290 " nop", \
291 /* LSE atomics */ \
292 " neg %[i], %[i]\n" \
293 " ldadd" #mb " %[i], x30, %[v]\n" \
294 " add %[i], %[i], x30") \
295 : [i] "+r" (x0), [v] "+Q" (v->counter) \
296 : "r" (x1) \
297 : "x30" , ##cl); \
298 \
299 return x0; \
300}
261 301
262 asm volatile(ARM64_LSE_ATOMIC_INSN( 302ATOMIC64_OP_SUB_RETURN(_relaxed, )
263 /* LL/SC */ 303ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
264 " nop\n" 304ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
265 __LL_SC_ATOMIC64(sub_return) 305ATOMIC64_OP_SUB_RETURN( , al, "memory")
266 " nop",
267 /* LSE atomics */
268 " neg %[i], %[i]\n"
269 " ldaddal %[i], x30, %[v]\n"
270 " add %[i], %[i], x30")
271 : [i] "+r" (x0), [v] "+Q" (v->counter)
272 : "r" (x1)
273 : "x30", "memory");
274 306
275 return x0; 307#undef ATOMIC64_OP_SUB_RETURN
276}
277 308
278static inline long atomic64_dec_if_positive(atomic64_t *v) 309static inline long atomic64_dec_if_positive(atomic64_t *v)
279{ 310{
@@ -333,14 +364,22 @@ static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
333 return x0; \ 364 return x0; \
334} 365}
335 366
336__CMPXCHG_CASE(w, b, 1, ) 367__CMPXCHG_CASE(w, b, 1, )
337__CMPXCHG_CASE(w, h, 2, ) 368__CMPXCHG_CASE(w, h, 2, )
338__CMPXCHG_CASE(w, , 4, ) 369__CMPXCHG_CASE(w, , 4, )
339__CMPXCHG_CASE(x, , 8, ) 370__CMPXCHG_CASE(x, , 8, )
340__CMPXCHG_CASE(w, b, mb_1, al, "memory") 371__CMPXCHG_CASE(w, b, acq_1, a, "memory")
341__CMPXCHG_CASE(w, h, mb_2, al, "memory") 372__CMPXCHG_CASE(w, h, acq_2, a, "memory")
342__CMPXCHG_CASE(w, , mb_4, al, "memory") 373__CMPXCHG_CASE(w, , acq_4, a, "memory")
343__CMPXCHG_CASE(x, , mb_8, al, "memory") 374__CMPXCHG_CASE(x, , acq_8, a, "memory")
375__CMPXCHG_CASE(w, b, rel_1, l, "memory")
376__CMPXCHG_CASE(w, h, rel_2, l, "memory")
377__CMPXCHG_CASE(w, , rel_4, l, "memory")
378__CMPXCHG_CASE(x, , rel_8, l, "memory")
379__CMPXCHG_CASE(w, b, mb_1, al, "memory")
380__CMPXCHG_CASE(w, h, mb_2, al, "memory")
381__CMPXCHG_CASE(w, , mb_4, al, "memory")
382__CMPXCHG_CASE(x, , mb_8, al, "memory")
344 383
345#undef __LL_SC_CMPXCHG 384#undef __LL_SC_CMPXCHG
346#undef __CMPXCHG_CASE 385#undef __CMPXCHG_CASE
@@ -348,7 +387,7 @@ __CMPXCHG_CASE(x, , mb_8, al, "memory")
348#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op) 387#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
349 388
350#define __CMPXCHG_DBL(name, mb, cl...) \ 389#define __CMPXCHG_DBL(name, mb, cl...) \
351static inline int __cmpxchg_double##name(unsigned long old1, \ 390static inline long __cmpxchg_double##name(unsigned long old1, \
352 unsigned long old2, \ 391 unsigned long old2, \
353 unsigned long new1, \ 392 unsigned long new1, \
354 unsigned long new2, \ 393 unsigned long new2, \
diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h
index 624f9679f4b0..9622eb48f894 100644
--- a/arch/arm64/include/asm/barrier.h
+++ b/arch/arm64/include/asm/barrier.h
@@ -64,27 +64,31 @@ do { \
64 64
65#define smp_load_acquire(p) \ 65#define smp_load_acquire(p) \
66({ \ 66({ \
67 typeof(*p) ___p1; \ 67 union { typeof(*p) __val; char __c[1]; } __u; \
68 compiletime_assert_atomic_type(*p); \ 68 compiletime_assert_atomic_type(*p); \
69 switch (sizeof(*p)) { \ 69 switch (sizeof(*p)) { \
70 case 1: \ 70 case 1: \
71 asm volatile ("ldarb %w0, %1" \ 71 asm volatile ("ldarb %w0, %1" \
72 : "=r" (___p1) : "Q" (*p) : "memory"); \ 72 : "=r" (*(__u8 *)__u.__c) \
73 : "Q" (*p) : "memory"); \
73 break; \ 74 break; \
74 case 2: \ 75 case 2: \
75 asm volatile ("ldarh %w0, %1" \ 76 asm volatile ("ldarh %w0, %1" \
76 : "=r" (___p1) : "Q" (*p) : "memory"); \ 77 : "=r" (*(__u16 *)__u.__c) \
78 : "Q" (*p) : "memory"); \
77 break; \ 79 break; \
78 case 4: \ 80 case 4: \
79 asm volatile ("ldar %w0, %1" \ 81 asm volatile ("ldar %w0, %1" \
80 : "=r" (___p1) : "Q" (*p) : "memory"); \ 82 : "=r" (*(__u32 *)__u.__c) \
83 : "Q" (*p) : "memory"); \
81 break; \ 84 break; \
82 case 8: \ 85 case 8: \
83 asm volatile ("ldar %0, %1" \ 86 asm volatile ("ldar %0, %1" \
84 : "=r" (___p1) : "Q" (*p) : "memory"); \ 87 : "=r" (*(__u64 *)__u.__c) \
88 : "Q" (*p) : "memory"); \
85 break; \ 89 break; \
86 } \ 90 } \
87 ___p1; \ 91 __u.__val; \
88}) 92})
89 93
90#define read_barrier_depends() do { } while(0) 94#define read_barrier_depends() do { } while(0)
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index bde449936e2f..5082b30bc2c0 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -18,7 +18,7 @@
18 18
19#include <asm/cachetype.h> 19#include <asm/cachetype.h>
20 20
21#define L1_CACHE_SHIFT 6 21#define L1_CACHE_SHIFT 7
22#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) 22#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
23 23
24/* 24/*
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index c75b8d027eb1..54efedaf331f 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -115,6 +115,13 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *,
115#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 115#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
116extern void flush_dcache_page(struct page *); 116extern void flush_dcache_page(struct page *);
117 117
118static inline void __local_flush_icache_all(void)
119{
120 asm("ic iallu");
121 dsb(nsh);
122 isb();
123}
124
118static inline void __flush_icache_all(void) 125static inline void __flush_icache_all(void)
119{ 126{
120 asm("ic ialluis"); 127 asm("ic ialluis");
diff --git a/arch/arm64/include/asm/cachetype.h b/arch/arm64/include/asm/cachetype.h
index da2fc9e3cedd..f5588692f1d4 100644
--- a/arch/arm64/include/asm/cachetype.h
+++ b/arch/arm64/include/asm/cachetype.h
@@ -34,8 +34,8 @@
34 34
35#define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK) 35#define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
36 36
37#define ICACHEF_ALIASING BIT(0) 37#define ICACHEF_ALIASING 0
38#define ICACHEF_AIVIVT BIT(1) 38#define ICACHEF_AIVIVT 1
39 39
40extern unsigned long __icache_flags; 40extern unsigned long __icache_flags;
41 41
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h
index 899e9f1d19e4..9ea611ea69df 100644
--- a/arch/arm64/include/asm/cmpxchg.h
+++ b/arch/arm64/include/asm/cmpxchg.h
@@ -25,154 +25,151 @@
25#include <asm/barrier.h> 25#include <asm/barrier.h>
26#include <asm/lse.h> 26#include <asm/lse.h>
27 27
28static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) 28/*
29{ 29 * We need separate acquire parameters for ll/sc and lse, since the full
30 unsigned long ret, tmp; 30 * barrier case is generated as release+dmb for the former and
31 31 * acquire+release for the latter.
32 switch (size) { 32 */
33 case 1: 33#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \
34 asm volatile(ARM64_LSE_ATOMIC_INSN( 34static inline unsigned long __xchg_case_##name(unsigned long x, \
35 /* LL/SC */ 35 volatile void *ptr) \
36 " prfm pstl1strm, %2\n" 36{ \
37 "1: ldxrb %w0, %2\n" 37 unsigned long ret, tmp; \
38 " stlxrb %w1, %w3, %2\n" 38 \
39 " cbnz %w1, 1b\n" 39 asm volatile(ARM64_LSE_ATOMIC_INSN( \
40 " dmb ish", 40 /* LL/SC */ \
41 /* LSE atomics */ 41 " prfm pstl1strm, %2\n" \
42 " nop\n" 42 "1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \
43 " nop\n" 43 " st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \
44 " swpalb %w3, %w0, %2\n" 44 " cbnz %w1, 1b\n" \
45 " nop\n" 45 " " #mb, \
46 " nop") 46 /* LSE atomics */ \
47 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) 47 " nop\n" \
48 : "r" (x) 48 " nop\n" \
49 : "memory"); 49 " swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \
50 break; 50 " nop\n" \
51 case 2: 51 " " #nop_lse) \
52 asm volatile(ARM64_LSE_ATOMIC_INSN( 52 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) \
53 /* LL/SC */ 53 : "r" (x) \
54 " prfm pstl1strm, %2\n" 54 : cl); \
55 "1: ldxrh %w0, %2\n" 55 \
56 " stlxrh %w1, %w3, %2\n" 56 return ret; \
57 " cbnz %w1, 1b\n"
58 " dmb ish",
59 /* LSE atomics */
60 " nop\n"
61 " nop\n"
62 " swpalh %w3, %w0, %2\n"
63 " nop\n"
64 " nop")
65 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
66 : "r" (x)
67 : "memory");
68 break;
69 case 4:
70 asm volatile(ARM64_LSE_ATOMIC_INSN(
71 /* LL/SC */
72 " prfm pstl1strm, %2\n"
73 "1: ldxr %w0, %2\n"
74 " stlxr %w1, %w3, %2\n"
75 " cbnz %w1, 1b\n"
76 " dmb ish",
77 /* LSE atomics */
78 " nop\n"
79 " nop\n"
80 " swpal %w3, %w0, %2\n"
81 " nop\n"
82 " nop")
83 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
84 : "r" (x)
85 : "memory");
86 break;
87 case 8:
88 asm volatile(ARM64_LSE_ATOMIC_INSN(
89 /* LL/SC */
90 " prfm pstl1strm, %2\n"
91 "1: ldxr %0, %2\n"
92 " stlxr %w1, %3, %2\n"
93 " cbnz %w1, 1b\n"
94 " dmb ish",
95 /* LSE atomics */
96 " nop\n"
97 " nop\n"
98 " swpal %3, %0, %2\n"
99 " nop\n"
100 " nop")
101 : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
102 : "r" (x)
103 : "memory");
104 break;
105 default:
106 BUILD_BUG();
107 }
108
109 return ret;
110} 57}
111 58
112#define xchg(ptr,x) \ 59__XCHG_CASE(w, b, 1, , , , , , )
113({ \ 60__XCHG_CASE(w, h, 2, , , , , , )
114 __typeof__(*(ptr)) __ret; \ 61__XCHG_CASE(w, , 4, , , , , , )
115 __ret = (__typeof__(*(ptr))) \ 62__XCHG_CASE( , , 8, , , , , , )
116 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ 63__XCHG_CASE(w, b, acq_1, , , a, a, , "memory")
117 __ret; \ 64__XCHG_CASE(w, h, acq_2, , , a, a, , "memory")
65__XCHG_CASE(w, , acq_4, , , a, a, , "memory")
66__XCHG_CASE( , , acq_8, , , a, a, , "memory")
67__XCHG_CASE(w, b, rel_1, , , , , l, "memory")
68__XCHG_CASE(w, h, rel_2, , , , , l, "memory")
69__XCHG_CASE(w, , rel_4, , , , , l, "memory")
70__XCHG_CASE( , , rel_8, , , , , l, "memory")
71__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory")
72__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory")
73__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory")
74__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory")
75
76#undef __XCHG_CASE
77
78#define __XCHG_GEN(sfx) \
79static inline unsigned long __xchg##sfx(unsigned long x, \
80 volatile void *ptr, \
81 int size) \
82{ \
83 switch (size) { \
84 case 1: \
85 return __xchg_case##sfx##_1(x, ptr); \
86 case 2: \
87 return __xchg_case##sfx##_2(x, ptr); \
88 case 4: \
89 return __xchg_case##sfx##_4(x, ptr); \
90 case 8: \
91 return __xchg_case##sfx##_8(x, ptr); \
92 default: \
93 BUILD_BUG(); \
94 } \
95 \
96 unreachable(); \
97}
98
99__XCHG_GEN()
100__XCHG_GEN(_acq)
101__XCHG_GEN(_rel)
102__XCHG_GEN(_mb)
103
104#undef __XCHG_GEN
105
106#define __xchg_wrapper(sfx, ptr, x) \
107({ \
108 __typeof__(*(ptr)) __ret; \
109 __ret = (__typeof__(*(ptr))) \
110 __xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
111 __ret; \
118}) 112})
119 113
120static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, 114/* xchg */
121 unsigned long new, int size) 115#define xchg_relaxed(...) __xchg_wrapper( , __VA_ARGS__)
122{ 116#define xchg_acquire(...) __xchg_wrapper(_acq, __VA_ARGS__)
123 switch (size) { 117#define xchg_release(...) __xchg_wrapper(_rel, __VA_ARGS__)
124 case 1: 118#define xchg(...) __xchg_wrapper( _mb, __VA_ARGS__)
125 return __cmpxchg_case_1(ptr, (u8)old, new); 119
126 case 2: 120#define __CMPXCHG_GEN(sfx) \
127 return __cmpxchg_case_2(ptr, (u16)old, new); 121static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
128 case 4: 122 unsigned long old, \
129 return __cmpxchg_case_4(ptr, old, new); 123 unsigned long new, \
130 case 8: 124 int size) \
131 return __cmpxchg_case_8(ptr, old, new); 125{ \
132 default: 126 switch (size) { \
133 BUILD_BUG(); 127 case 1: \
134 } 128 return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \
135 129 case 2: \
136 unreachable(); 130 return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \
131 case 4: \
132 return __cmpxchg_case##sfx##_4(ptr, old, new); \
133 case 8: \
134 return __cmpxchg_case##sfx##_8(ptr, old, new); \
135 default: \
136 BUILD_BUG(); \
137 } \
138 \
139 unreachable(); \
137} 140}
138 141
139static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, 142__CMPXCHG_GEN()
140 unsigned long new, int size) 143__CMPXCHG_GEN(_acq)
141{ 144__CMPXCHG_GEN(_rel)
142 switch (size) { 145__CMPXCHG_GEN(_mb)
143 case 1:
144 return __cmpxchg_case_mb_1(ptr, (u8)old, new);
145 case 2:
146 return __cmpxchg_case_mb_2(ptr, (u16)old, new);
147 case 4:
148 return __cmpxchg_case_mb_4(ptr, old, new);
149 case 8:
150 return __cmpxchg_case_mb_8(ptr, old, new);
151 default:
152 BUILD_BUG();
153 }
154
155 unreachable();
156}
157 146
158#define cmpxchg(ptr, o, n) \ 147#undef __CMPXCHG_GEN
159({ \
160 __typeof__(*(ptr)) __ret; \
161 __ret = (__typeof__(*(ptr))) \
162 __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
163 sizeof(*(ptr))); \
164 __ret; \
165})
166 148
167#define cmpxchg_local(ptr, o, n) \ 149#define __cmpxchg_wrapper(sfx, ptr, o, n) \
168({ \ 150({ \
169 __typeof__(*(ptr)) __ret; \ 151 __typeof__(*(ptr)) __ret; \
170 __ret = (__typeof__(*(ptr))) \ 152 __ret = (__typeof__(*(ptr))) \
171 __cmpxchg((ptr), (unsigned long)(o), \ 153 __cmpxchg##sfx((ptr), (unsigned long)(o), \
172 (unsigned long)(n), sizeof(*(ptr))); \ 154 (unsigned long)(n), sizeof(*(ptr))); \
173 __ret; \ 155 __ret; \
174}) 156})
175 157
158/* cmpxchg */
159#define cmpxchg_relaxed(...) __cmpxchg_wrapper( , __VA_ARGS__)
160#define cmpxchg_acquire(...) __cmpxchg_wrapper(_acq, __VA_ARGS__)
161#define cmpxchg_release(...) __cmpxchg_wrapper(_rel, __VA_ARGS__)
162#define cmpxchg(...) __cmpxchg_wrapper( _mb, __VA_ARGS__)
163#define cmpxchg_local cmpxchg_relaxed
164
165/* cmpxchg64 */
166#define cmpxchg64_relaxed cmpxchg_relaxed
167#define cmpxchg64_acquire cmpxchg_acquire
168#define cmpxchg64_release cmpxchg_release
169#define cmpxchg64 cmpxchg
170#define cmpxchg64_local cmpxchg_local
171
172/* cmpxchg_double */
176#define system_has_cmpxchg_double() 1 173#define system_has_cmpxchg_double() 1
177 174
178#define __cmpxchg_double_check(ptr1, ptr2) \ 175#define __cmpxchg_double_check(ptr1, ptr2) \
@@ -202,6 +199,7 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
202 __ret; \ 199 __ret; \
203}) 200})
204 201
202/* this_cpu_cmpxchg */
205#define _protect_cmpxchg_local(pcp, o, n) \ 203#define _protect_cmpxchg_local(pcp, o, n) \
206({ \ 204({ \
207 typeof(*raw_cpu_ptr(&(pcp))) __ret; \ 205 typeof(*raw_cpu_ptr(&(pcp))) __ret; \
@@ -227,9 +225,4 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
227 __ret; \ 225 __ret; \
228}) 226})
229 227
230#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
231#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
232
233#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n))
234
235#endif /* __ASM_CMPXCHG_H */ 228#endif /* __ASM_CMPXCHG_H */
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h
index 7fbed6919b54..eb8432bb82b8 100644
--- a/arch/arm64/include/asm/compat.h
+++ b/arch/arm64/include/asm/compat.h
@@ -23,7 +23,6 @@
23 */ 23 */
24#include <linux/types.h> 24#include <linux/types.h>
25#include <linux/sched.h> 25#include <linux/sched.h>
26#include <linux/ptrace.h>
27 26
28#define COMPAT_USER_HZ 100 27#define COMPAT_USER_HZ 100
29#ifdef __AARCH64EB__ 28#ifdef __AARCH64EB__
@@ -234,7 +233,7 @@ static inline compat_uptr_t ptr_to_compat(void __user *uptr)
234 return (u32)(unsigned long)uptr; 233 return (u32)(unsigned long)uptr;
235} 234}
236 235
237#define compat_user_stack_pointer() (user_stack_pointer(current_pt_regs())) 236#define compat_user_stack_pointer() (user_stack_pointer(task_pt_regs(current)))
238 237
239static inline void __user *arch_compat_alloc_user_space(long len) 238static inline void __user *arch_compat_alloc_user_space(long len)
240{ 239{
diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h
index 8e797b2fcc01..b5e9cee4b5f8 100644
--- a/arch/arm64/include/asm/cpu.h
+++ b/arch/arm64/include/asm/cpu.h
@@ -63,4 +63,8 @@ DECLARE_PER_CPU(struct cpuinfo_arm64, cpu_data);
63void cpuinfo_store_cpu(void); 63void cpuinfo_store_cpu(void);
64void __init cpuinfo_store_boot_cpu(void); 64void __init cpuinfo_store_boot_cpu(void);
65 65
66void __init init_cpu_features(struct cpuinfo_arm64 *info);
67void update_cpu_features(int cpu, struct cpuinfo_arm64 *info,
68 struct cpuinfo_arm64 *boot);
69
66#endif /* __ASM_CPU_H */ 70#endif /* __ASM_CPU_H */
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 171570702bb8..52722ee73dba 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -10,6 +10,7 @@
10#define __ASM_CPUFEATURE_H 10#define __ASM_CPUFEATURE_H
11 11
12#include <asm/hwcap.h> 12#include <asm/hwcap.h>
13#include <asm/sysreg.h>
13 14
14/* 15/*
15 * In the arm64 world (as in the ARM world), elf_hwcap is used both internally 16 * In the arm64 world (as in the ARM world), elf_hwcap is used both internally
@@ -27,18 +28,51 @@
27#define ARM64_HAS_SYSREG_GIC_CPUIF 3 28#define ARM64_HAS_SYSREG_GIC_CPUIF 3
28#define ARM64_HAS_PAN 4 29#define ARM64_HAS_PAN 4
29#define ARM64_HAS_LSE_ATOMICS 5 30#define ARM64_HAS_LSE_ATOMICS 5
31#define ARM64_WORKAROUND_CAVIUM_23154 6
32#define ARM64_WORKAROUND_834220 7
30 33
31#define ARM64_NCAPS 6 34#define ARM64_NCAPS 8
32 35
33#ifndef __ASSEMBLY__ 36#ifndef __ASSEMBLY__
34 37
35#include <linux/kernel.h> 38#include <linux/kernel.h>
36 39
40/* CPU feature register tracking */
41enum ftr_type {
42 FTR_EXACT, /* Use a predefined safe value */
43 FTR_LOWER_SAFE, /* Smaller value is safe */
44 FTR_HIGHER_SAFE,/* Bigger value is safe */
45};
46
47#define FTR_STRICT true /* SANITY check strict matching required */
48#define FTR_NONSTRICT false /* SANITY check ignored */
49
50struct arm64_ftr_bits {
51 bool strict; /* CPU Sanity check: strict matching required ? */
52 enum ftr_type type;
53 u8 shift;
54 u8 width;
55 s64 safe_val; /* safe value for discrete features */
56};
57
58/*
59 * @arm64_ftr_reg - Feature register
60 * @strict_mask Bits which should match across all CPUs for sanity.
61 * @sys_val Safe value across the CPUs (system view)
62 */
63struct arm64_ftr_reg {
64 u32 sys_id;
65 const char *name;
66 u64 strict_mask;
67 u64 sys_val;
68 struct arm64_ftr_bits *ftr_bits;
69};
70
37struct arm64_cpu_capabilities { 71struct arm64_cpu_capabilities {
38 const char *desc; 72 const char *desc;
39 u16 capability; 73 u16 capability;
40 bool (*matches)(const struct arm64_cpu_capabilities *); 74 bool (*matches)(const struct arm64_cpu_capabilities *);
41 void (*enable)(void); 75 void (*enable)(void *); /* Called on all active CPUs */
42 union { 76 union {
43 struct { /* To be used for erratum handling only */ 77 struct { /* To be used for erratum handling only */
44 u32 midr_model; 78 u32 midr_model;
@@ -46,8 +80,11 @@ struct arm64_cpu_capabilities {
46 }; 80 };
47 81
48 struct { /* Feature register checking */ 82 struct { /* Feature register checking */
83 u32 sys_reg;
49 int field_pos; 84 int field_pos;
50 int min_field_value; 85 int min_field_value;
86 int hwcap_type;
87 unsigned long hwcap;
51 }; 88 };
52 }; 89 };
53}; 90};
@@ -75,19 +112,59 @@ static inline void cpus_set_cap(unsigned int num)
75 __set_bit(num, cpu_hwcaps); 112 __set_bit(num, cpu_hwcaps);
76} 113}
77 114
78static inline int __attribute_const__ cpuid_feature_extract_field(u64 features, 115static inline int __attribute_const__
79 int field) 116cpuid_feature_extract_field_width(u64 features, int field, int width)
117{
118 return (s64)(features << (64 - width - field)) >> (64 - width);
119}
120
121static inline int __attribute_const__
122cpuid_feature_extract_field(u64 features, int field)
123{
124 return cpuid_feature_extract_field_width(features, field, 4);
125}
126
127static inline u64 arm64_ftr_mask(struct arm64_ftr_bits *ftrp)
128{
129 return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift);
130}
131
132static inline s64 arm64_ftr_value(struct arm64_ftr_bits *ftrp, u64 val)
133{
134 return cpuid_feature_extract_field_width(val, ftrp->shift, ftrp->width);
135}
136
137static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
80{ 138{
81 return (s64)(features << (64 - 4 - field)) >> (64 - 4); 139 return cpuid_feature_extract_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 ||
140 cpuid_feature_extract_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1;
82} 141}
83 142
143void __init setup_cpu_features(void);
84 144
85void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, 145void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
86 const char *info); 146 const char *info);
87void check_local_cpu_errata(void); 147void check_local_cpu_errata(void);
88void check_local_cpu_features(void); 148
89bool cpu_supports_mixed_endian_el0(void); 149#ifdef CONFIG_HOTPLUG_CPU
90bool system_supports_mixed_endian_el0(void); 150void verify_local_cpu_capabilities(void);
151#else
152static inline void verify_local_cpu_capabilities(void)
153{
154}
155#endif
156
157u64 read_system_reg(u32 id);
158
159static inline bool cpu_supports_mixed_endian_el0(void)
160{
161 return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
162}
163
164static inline bool system_supports_mixed_endian_el0(void)
165{
166 return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1));
167}
91 168
92#endif /* __ASSEMBLY__ */ 169#endif /* __ASSEMBLY__ */
93 170
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index ee6403df9fe4..1a5949364ed0 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -62,24 +62,18 @@
62 (0xf << MIDR_ARCHITECTURE_SHIFT) | \ 62 (0xf << MIDR_ARCHITECTURE_SHIFT) | \
63 ((partnum) << MIDR_PARTNUM_SHIFT)) 63 ((partnum) << MIDR_PARTNUM_SHIFT))
64 64
65#define ARM_CPU_IMP_ARM 0x41 65#define ARM_CPU_IMP_ARM 0x41
66#define ARM_CPU_IMP_APM 0x50 66#define ARM_CPU_IMP_APM 0x50
67#define ARM_CPU_IMP_CAVIUM 0x43
67 68
68#define ARM_CPU_PART_AEM_V8 0xD0F 69#define ARM_CPU_PART_AEM_V8 0xD0F
69#define ARM_CPU_PART_FOUNDATION 0xD00 70#define ARM_CPU_PART_FOUNDATION 0xD00
70#define ARM_CPU_PART_CORTEX_A57 0xD07 71#define ARM_CPU_PART_CORTEX_A57 0xD07
71#define ARM_CPU_PART_CORTEX_A53 0xD03 72#define ARM_CPU_PART_CORTEX_A53 0xD03
72 73
73#define APM_CPU_PART_POTENZA 0x000 74#define APM_CPU_PART_POTENZA 0x000
74 75
75#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16 76#define CAVIUM_CPU_PART_THUNDERX 0x0A1
76#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT)
77#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \
78 (((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT)
79#define ID_AA64MMFR0_BIGEND_SHIFT 8
80#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT)
81#define ID_AA64MMFR0_BIGEND(mmfr0) \
82 (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)
83 77
84#ifndef __ASSEMBLY__ 78#ifndef __ASSEMBLY__
85 79
@@ -112,12 +106,6 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void)
112{ 106{
113 return read_cpuid(CTR_EL0); 107 return read_cpuid(CTR_EL0);
114} 108}
115
116static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
117{
118 return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) ||
119 (ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1);
120}
121#endif /* __ASSEMBLY__ */ 109#endif /* __ASSEMBLY__ */
122 110
123#endif 111#endif
diff --git a/arch/arm64/include/asm/dcc.h b/arch/arm64/include/asm/dcc.h
new file mode 100644
index 000000000000..65e0190e97c8
--- /dev/null
+++ b/arch/arm64/include/asm/dcc.h
@@ -0,0 +1,55 @@
1/* Copyright (c) 2014-2015 The Linux Foundation. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * A call to __dcc_getchar() or __dcc_putchar() is typically followed by
13 * a call to __dcc_getstatus(). We want to make sure that the CPU does
14 * not speculative read the DCC status before executing the read or write
15 * instruction. That's what the ISBs are for.
16 *
17 * The 'volatile' ensures that the compiler does not cache the status bits,
18 * and instead reads the DCC register every time.
19 */
20#ifndef __ASM_DCC_H
21#define __ASM_DCC_H
22
23#include <asm/barrier.h>
24
25static inline u32 __dcc_getstatus(void)
26{
27 u32 ret;
28
29 asm volatile("mrs %0, mdccsr_el0" : "=r" (ret));
30
31 return ret;
32}
33
34static inline char __dcc_getchar(void)
35{
36 char c;
37
38 asm volatile("mrs %0, dbgdtrrx_el0" : "=r" (c));
39 isb();
40
41 return c;
42}
43
44static inline void __dcc_putchar(char c)
45{
46 /*
47 * The typecast is to make absolutely certain that 'c' is
48 * zero-extended.
49 */
50 asm volatile("msr dbgdtrtx_el0, %0"
51 : : "r" ((unsigned long)(unsigned char)c));
52 isb();
53}
54
55#endif
diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h
index cfdb34bedbcd..61e08f360e31 100644
--- a/arch/arm64/include/asm/dma-mapping.h
+++ b/arch/arm64/include/asm/dma-mapping.h
@@ -18,7 +18,6 @@
18 18
19#ifdef __KERNEL__ 19#ifdef __KERNEL__
20 20
21#include <linux/acpi.h>
22#include <linux/types.h> 21#include <linux/types.h>
23#include <linux/vmalloc.h> 22#include <linux/vmalloc.h>
24 23
@@ -26,22 +25,16 @@
26#include <asm/xen/hypervisor.h> 25#include <asm/xen/hypervisor.h>
27 26
28#define DMA_ERROR_CODE (~(dma_addr_t)0) 27#define DMA_ERROR_CODE (~(dma_addr_t)0)
29extern struct dma_map_ops *dma_ops;
30extern struct dma_map_ops dummy_dma_ops; 28extern struct dma_map_ops dummy_dma_ops;
31 29
32static inline struct dma_map_ops *__generic_dma_ops(struct device *dev) 30static inline struct dma_map_ops *__generic_dma_ops(struct device *dev)
33{ 31{
34 if (unlikely(!dev)) 32 if (dev && dev->archdata.dma_ops)
35 return dma_ops;
36 else if (dev->archdata.dma_ops)
37 return dev->archdata.dma_ops; 33 return dev->archdata.dma_ops;
38 else if (acpi_disabled)
39 return dma_ops;
40 34
41 /* 35 /*
42 * When ACPI is enabled, if arch_set_dma_ops is not called, 36 * We expect no ISA devices, and all other DMA masters are expected to
43 * we will disable device DMA capability by setting it 37 * have someone call arch_setup_dma_ops at device creation time.
44 * to dummy_dma_ops.
45 */ 38 */
46 return &dummy_dma_ops; 39 return &dummy_dma_ops;
47} 40}
@@ -54,16 +47,15 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev)
54 return __generic_dma_ops(dev); 47 return __generic_dma_ops(dev);
55} 48}
56 49
57static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 50void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
58 struct iommu_ops *iommu, bool coherent) 51 struct iommu_ops *iommu, bool coherent);
59{
60 if (!acpi_disabled && !dev->archdata.dma_ops)
61 dev->archdata.dma_ops = dma_ops;
62
63 dev->archdata.dma_coherent = coherent;
64}
65#define arch_setup_dma_ops arch_setup_dma_ops 52#define arch_setup_dma_ops arch_setup_dma_ops
66 53
54#ifdef CONFIG_IOMMU_DMA
55void arch_teardown_dma_ops(struct device *dev);
56#define arch_teardown_dma_ops arch_teardown_dma_ops
57#endif
58
67/* do not use this function in a driver */ 59/* do not use this function in a driver */
68static inline bool is_device_dma_coherent(struct device *dev) 60static inline bool is_device_dma_coherent(struct device *dev)
69{ 61{
diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 8b9884c726ad..309704544d22 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -17,6 +17,7 @@
17 17
18#ifndef __ASSEMBLY__ 18#ifndef __ASSEMBLY__
19#include <linux/kernel.h> 19#include <linux/kernel.h>
20#include <linux/sizes.h>
20#include <asm/boot.h> 21#include <asm/boot.h>
21#include <asm/page.h> 22#include <asm/page.h>
22 23
@@ -55,11 +56,7 @@ enum fixed_addresses {
55 * Temporary boot-time mappings, used by early_ioremap(), 56 * Temporary boot-time mappings, used by early_ioremap(),
56 * before ioremap() is functional. 57 * before ioremap() is functional.
57 */ 58 */
58#ifdef CONFIG_ARM64_64K_PAGES 59#define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE)
59#define NR_FIX_BTMAPS 4
60#else
61#define NR_FIX_BTMAPS 64
62#endif
63#define FIX_BTMAPS_SLOTS 7 60#define FIX_BTMAPS_SLOTS 7
64#define TOTAL_FIX_BTMAPS (NR_FIX_BTMAPS * FIX_BTMAPS_SLOTS) 61#define TOTAL_FIX_BTMAPS (NR_FIX_BTMAPS * FIX_BTMAPS_SLOTS)
65 62
diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
index 4c47cb2fbb52..e54415ec6935 100644
--- a/arch/arm64/include/asm/hw_breakpoint.h
+++ b/arch/arm64/include/asm/hw_breakpoint.h
@@ -17,6 +17,7 @@
17#define __ASM_HW_BREAKPOINT_H 17#define __ASM_HW_BREAKPOINT_H
18 18
19#include <asm/cputype.h> 19#include <asm/cputype.h>
20#include <asm/cpufeature.h>
20 21
21#ifdef __KERNEL__ 22#ifdef __KERNEL__
22 23
@@ -137,13 +138,17 @@ extern struct pmu perf_ops_bp;
137/* Determine number of BRP registers available. */ 138/* Determine number of BRP registers available. */
138static inline int get_num_brps(void) 139static inline int get_num_brps(void)
139{ 140{
140 return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1; 141 return 1 +
142 cpuid_feature_extract_field(read_system_reg(SYS_ID_AA64DFR0_EL1),
143 ID_AA64DFR0_BRPS_SHIFT);
141} 144}
142 145
143/* Determine number of WRP registers available. */ 146/* Determine number of WRP registers available. */
144static inline int get_num_wrps(void) 147static inline int get_num_wrps(void)
145{ 148{
146 return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1; 149 return 1 +
150 cpuid_feature_extract_field(read_system_reg(SYS_ID_AA64DFR0_EL1),
151 ID_AA64DFR0_WRPS_SHIFT);
147} 152}
148 153
149#endif /* __KERNEL__ */ 154#endif /* __KERNEL__ */
diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
index 0ad735166d9f..400b80b49595 100644
--- a/arch/arm64/include/asm/hwcap.h
+++ b/arch/arm64/include/asm/hwcap.h
@@ -52,6 +52,14 @@
52extern unsigned int compat_elf_hwcap, compat_elf_hwcap2; 52extern unsigned int compat_elf_hwcap, compat_elf_hwcap2;
53#endif 53#endif
54 54
55enum {
56 CAP_HWCAP = 1,
57#ifdef CONFIG_COMPAT
58 CAP_COMPAT_HWCAP,
59 CAP_COMPAT_HWCAP2,
60#endif
61};
62
55extern unsigned long elf_hwcap; 63extern unsigned long elf_hwcap;
56#endif 64#endif
57#endif 65#endif
diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h
index 8b9bf54105b3..8e8d30684392 100644
--- a/arch/arm64/include/asm/irq.h
+++ b/arch/arm64/include/asm/irq.h
@@ -1,26 +1,12 @@
1#ifndef __ASM_IRQ_H 1#ifndef __ASM_IRQ_H
2#define __ASM_IRQ_H 2#define __ASM_IRQ_H
3 3
4#include <linux/irqchip/arm-gic-acpi.h>
5
6#include <asm-generic/irq.h> 4#include <asm-generic/irq.h>
7 5
8struct pt_regs; 6struct pt_regs;
9 7
10extern void migrate_irqs(void);
11extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); 8extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
12 9
13static inline void acpi_irq_init(void)
14{
15 /*
16 * Hardcode ACPI IRQ chip initialization to GICv2 for now.
17 * Proper irqchip infrastructure will be implemented along with
18 * incoming GICv2m|GICv3|ITS bits.
19 */
20 acpi_gic_init();
21}
22#define acpi_irq_init acpi_irq_init
23
24static inline int nr_legacy_irqs(void) 10static inline int nr_legacy_irqs(void)
25{ 11{
26 return 0; 12 return 0;
diff --git a/arch/arm64/include/asm/kasan.h b/arch/arm64/include/asm/kasan.h
new file mode 100644
index 000000000000..2774fa384c47
--- /dev/null
+++ b/arch/arm64/include/asm/kasan.h
@@ -0,0 +1,38 @@
1#ifndef __ASM_KASAN_H
2#define __ASM_KASAN_H
3
4#ifndef __ASSEMBLY__
5
6#ifdef CONFIG_KASAN
7
8#include <linux/linkage.h>
9#include <asm/memory.h>
10
11/*
12 * KASAN_SHADOW_START: beginning of the kernel virtual addresses.
13 * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/8 of kernel virtual addresses.
14 */
15#define KASAN_SHADOW_START (VA_START)
16#define KASAN_SHADOW_END (KASAN_SHADOW_START + (1UL << (VA_BITS - 3)))
17
18/*
19 * This value is used to map an address to the corresponding shadow
20 * address by the following formula:
21 * shadow_addr = (address >> 3) + KASAN_SHADOW_OFFSET;
22 *
23 * (1 << 61) shadow addresses - [KASAN_SHADOW_OFFSET,KASAN_SHADOW_END]
24 * cover all 64-bits of virtual addresses. So KASAN_SHADOW_OFFSET
25 * should satisfy the following equation:
26 * KASAN_SHADOW_OFFSET = KASAN_SHADOW_END - (1ULL << 61)
27 */
28#define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << (64 - 3)))
29
30void kasan_init(void);
31asmlinkage void kasan_early_init(void);
32
33#else
34static inline void kasan_init(void) { }
35#endif
36
37#endif
38#endif
diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
new file mode 100644
index 000000000000..a459714ee29e
--- /dev/null
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -0,0 +1,83 @@
1/*
2 * Kernel page table mapping
3 *
4 * Copyright (C) 2015 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#ifndef __ASM_KERNEL_PGTABLE_H
20#define __ASM_KERNEL_PGTABLE_H
21
22
23/*
24 * The linear mapping and the start of memory are both 2M aligned (per
25 * the arm64 booting.txt requirements). Hence we can use section mapping
26 * with 4K (section size = 2M) but not with 16K (section size = 32M) or
27 * 64K (section size = 512M).
28 */
29#ifdef CONFIG_ARM64_4K_PAGES
30#define ARM64_SWAPPER_USES_SECTION_MAPS 1
31#else
32#define ARM64_SWAPPER_USES_SECTION_MAPS 0
33#endif
34
35/*
36 * The idmap and swapper page tables need some space reserved in the kernel
37 * image. Both require pgd, pud (4 levels only) and pmd tables to (section)
38 * map the kernel. With the 64K page configuration, swapper and idmap need to
39 * map to pte level. The swapper also maps the FDT (see __create_page_tables
40 * for more information). Note that the number of ID map translation levels
41 * could be increased on the fly if system RAM is out of reach for the default
42 * VA range, so pages required to map highest possible PA are reserved in all
43 * cases.
44 */
45#if ARM64_SWAPPER_USES_SECTION_MAPS
46#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
47#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT) - 1)
48#else
49#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
50#define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT))
51#endif
52
53#define SWAPPER_DIR_SIZE (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
54#define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
55
56/* Initial memory map size */
57#if ARM64_SWAPPER_USES_SECTION_MAPS
58#define SWAPPER_BLOCK_SHIFT SECTION_SHIFT
59#define SWAPPER_BLOCK_SIZE SECTION_SIZE
60#define SWAPPER_TABLE_SHIFT PUD_SHIFT
61#else
62#define SWAPPER_BLOCK_SHIFT PAGE_SHIFT
63#define SWAPPER_BLOCK_SIZE PAGE_SIZE
64#define SWAPPER_TABLE_SHIFT PMD_SHIFT
65#endif
66
67/* The size of the initial kernel direct mapping */
68#define SWAPPER_INIT_MAP_SIZE (_AC(1, UL) << SWAPPER_TABLE_SHIFT)
69
70/*
71 * Initial memory map attributes.
72 */
73#define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
74#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
75
76#if ARM64_SWAPPER_USES_SECTION_MAPS
77#define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
78#else
79#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
80#endif
81
82
83#endif /* __ASM_KERNEL_PGTABLE_H */
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 9694f2654593..5e6857b6bdc4 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -200,4 +200,20 @@
200/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ 200/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
201#define HPFAR_MASK (~UL(0xf)) 201#define HPFAR_MASK (~UL(0xf))
202 202
203#define kvm_arm_exception_type \
204 {0, "IRQ" }, \
205 {1, "TRAP" }
206
207#define ECN(x) { ESR_ELx_EC_##x, #x }
208
209#define kvm_arm_exception_class \
210 ECN(UNKNOWN), ECN(WFx), ECN(CP15_32), ECN(CP15_64), ECN(CP14_MR), \
211 ECN(CP14_LS), ECN(FP_ASIMD), ECN(CP10_ID), ECN(CP14_64), ECN(SVC64), \
212 ECN(HVC64), ECN(SMC64), ECN(SYS64), ECN(IMP_DEF), ECN(IABT_LOW), \
213 ECN(IABT_CUR), ECN(PC_ALIGN), ECN(DABT_LOW), ECN(DABT_CUR), \
214 ECN(SP_ALIGN), ECN(FP_EXC32), ECN(FP_EXC64), ECN(SERROR), \
215 ECN(BREAKPT_LOW), ECN(BREAKPT_CUR), ECN(SOFTSTP_LOW), \
216 ECN(SOFTSTP_CUR), ECN(WATCHPT_LOW), ECN(WATCHPT_CUR), \
217 ECN(BKPT32), ECN(VECTOR32), ECN(BRK64)
218
203#endif /* __ARM64_KVM_ARM_H__ */ 219#endif /* __ARM64_KVM_ARM_H__ */
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 17e92f05b1fe..3ca894ecf699 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -99,11 +99,13 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
99 *vcpu_cpsr(vcpu) |= COMPAT_PSR_T_BIT; 99 *vcpu_cpsr(vcpu) |= COMPAT_PSR_T_BIT;
100} 100}
101 101
102/*
103 * vcpu_reg should always be passed a register number coming from a
104 * read of ESR_EL2. Otherwise, it may give the wrong result on AArch32
105 * with banked registers.
106 */
102static inline unsigned long *vcpu_reg(const struct kvm_vcpu *vcpu, u8 reg_num) 107static inline unsigned long *vcpu_reg(const struct kvm_vcpu *vcpu, u8 reg_num)
103{ 108{
104 if (vcpu_mode_is_32bit(vcpu))
105 return vcpu_reg32(vcpu, reg_num);
106
107 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.regs[reg_num]; 109 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.regs[reg_num];
108} 110}
109 111
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index ed039688c221..a35ce7266aac 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -149,7 +149,10 @@ struct kvm_vcpu_arch {
149 u32 mdscr_el1; 149 u32 mdscr_el1;
150 } guest_debug_preserved; 150 } guest_debug_preserved;
151 151
152 /* Don't run the guest */ 152 /* vcpu power-off state */
153 bool power_off;
154
155 /* Don't run the guest (internal implementation need) */
153 bool pause; 156 bool pause;
154 157
155 /* IO related fields */ 158 /* IO related fields */
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 6b4c3ad75a2a..853953cd1f08 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -42,12 +42,14 @@
42 * PAGE_OFFSET - the virtual address of the start of the kernel image (top 42 * PAGE_OFFSET - the virtual address of the start of the kernel image (top
43 * (VA_BITS - 1)) 43 * (VA_BITS - 1))
44 * VA_BITS - the maximum number of bits for virtual addresses. 44 * VA_BITS - the maximum number of bits for virtual addresses.
45 * VA_START - the first kernel virtual address.
45 * TASK_SIZE - the maximum size of a user space task. 46 * TASK_SIZE - the maximum size of a user space task.
46 * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area. 47 * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
47 * The module space lives between the addresses given by TASK_SIZE 48 * The module space lives between the addresses given by TASK_SIZE
48 * and PAGE_OFFSET - it must be within 128MB of the kernel text. 49 * and PAGE_OFFSET - it must be within 128MB of the kernel text.
49 */ 50 */
50#define VA_BITS (CONFIG_ARM64_VA_BITS) 51#define VA_BITS (CONFIG_ARM64_VA_BITS)
52#define VA_START (UL(0xffffffffffffffff) << VA_BITS)
51#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) 53#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
52#define MODULES_END (PAGE_OFFSET) 54#define MODULES_END (PAGE_OFFSET)
53#define MODULES_VADDR (MODULES_END - SZ_64M) 55#define MODULES_VADDR (MODULES_END - SZ_64M)
@@ -68,10 +70,6 @@
68 70
69#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4)) 71#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
70 72
71#if TASK_SIZE_64 > MODULES_VADDR
72#error Top of 64-bit user space clashes with start of module space
73#endif
74
75/* 73/*
76 * Physical vs virtual RAM address space conversion. These are 74 * Physical vs virtual RAM address space conversion. These are
77 * private definitions which should NOT be used outside memory.h 75 * private definitions which should NOT be used outside memory.h
@@ -94,6 +92,7 @@
94#define MT_DEVICE_GRE 2 92#define MT_DEVICE_GRE 2
95#define MT_NORMAL_NC 3 93#define MT_NORMAL_NC 3
96#define MT_NORMAL 4 94#define MT_NORMAL 4
95#define MT_NORMAL_WT 5
97 96
98/* 97/*
99 * Memory types for Stage-2 translation 98 * Memory types for Stage-2 translation
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 030208767185..990124a67eeb 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -17,15 +17,16 @@
17#define __ASM_MMU_H 17#define __ASM_MMU_H
18 18
19typedef struct { 19typedef struct {
20 unsigned int id; 20 atomic64_t id;
21 raw_spinlock_t id_lock; 21 void *vdso;
22 void *vdso;
23} mm_context_t; 22} mm_context_t;
24 23
25#define INIT_MM_CONTEXT(name) \ 24/*
26 .context.id_lock = __RAW_SPIN_LOCK_UNLOCKED(name.context.id_lock), 25 * This macro is only used by the TLBI code, which cannot race with an
27 26 * ASID change and therefore doesn't need to reload the counter using
28#define ASID(mm) ((mm)->context.id & 0xffff) 27 * atomic64_read.
28 */
29#define ASID(mm) ((mm)->context.id.counter & 0xffff)
29 30
30extern void paging_init(void); 31extern void paging_init(void);
31extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); 32extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 8ec41e5f56f0..24165784b803 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -28,13 +28,6 @@
28#include <asm/cputype.h> 28#include <asm/cputype.h>
29#include <asm/pgtable.h> 29#include <asm/pgtable.h>
30 30
31#define MAX_ASID_BITS 16
32
33extern unsigned int cpu_last_asid;
34
35void __init_new_context(struct task_struct *tsk, struct mm_struct *mm);
36void __new_context(struct mm_struct *mm);
37
38#ifdef CONFIG_PID_IN_CONTEXTIDR 31#ifdef CONFIG_PID_IN_CONTEXTIDR
39static inline void contextidr_thread_switch(struct task_struct *next) 32static inline void contextidr_thread_switch(struct task_struct *next)
40{ 33{
@@ -77,96 +70,38 @@ static inline bool __cpu_uses_extended_idmap(void)
77 unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS))); 70 unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS)));
78} 71}
79 72
80static inline void __cpu_set_tcr_t0sz(u64 t0sz)
81{
82 unsigned long tcr;
83
84 if (__cpu_uses_extended_idmap())
85 asm volatile (
86 " mrs %0, tcr_el1 ;"
87 " bfi %0, %1, %2, %3 ;"
88 " msr tcr_el1, %0 ;"
89 " isb"
90 : "=&r" (tcr)
91 : "r"(t0sz), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH));
92}
93
94/*
95 * Set TCR.T0SZ to the value appropriate for activating the identity map.
96 */
97static inline void cpu_set_idmap_tcr_t0sz(void)
98{
99 __cpu_set_tcr_t0sz(idmap_t0sz);
100}
101
102/* 73/*
103 * Set TCR.T0SZ to its default value (based on VA_BITS) 74 * Set TCR.T0SZ to its default value (based on VA_BITS)
104 */ 75 */
105static inline void cpu_set_default_tcr_t0sz(void) 76static inline void cpu_set_default_tcr_t0sz(void)
106{ 77{
107 __cpu_set_tcr_t0sz(TCR_T0SZ(VA_BITS)); 78 unsigned long tcr;
108}
109
110static inline void switch_new_context(struct mm_struct *mm)
111{
112 unsigned long flags;
113
114 __new_context(mm);
115 79
116 local_irq_save(flags); 80 if (!__cpu_uses_extended_idmap())
117 cpu_switch_mm(mm->pgd, mm); 81 return;
118 local_irq_restore(flags);
119}
120 82
121static inline void check_and_switch_context(struct mm_struct *mm, 83 asm volatile (
122 struct task_struct *tsk) 84 " mrs %0, tcr_el1 ;"
123{ 85 " bfi %0, %1, %2, %3 ;"
124 /* 86 " msr tcr_el1, %0 ;"
125 * Required during context switch to avoid speculative page table 87 " isb"
126 * walking with the wrong TTBR. 88 : "=&r" (tcr)
127 */ 89 : "r"(TCR_T0SZ(VA_BITS)), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH));
128 cpu_set_reserved_ttbr0();
129
130 if (!((mm->context.id ^ cpu_last_asid) >> MAX_ASID_BITS))
131 /*
132 * The ASID is from the current generation, just switch to the
133 * new pgd. This condition is only true for calls from
134 * context_switch() and interrupts are already disabled.
135 */
136 cpu_switch_mm(mm->pgd, mm);
137 else if (irqs_disabled())
138 /*
139 * Defer the new ASID allocation until after the context
140 * switch critical region since __new_context() cannot be
141 * called with interrupts disabled.
142 */
143 set_ti_thread_flag(task_thread_info(tsk), TIF_SWITCH_MM);
144 else
145 /*
146 * That is a direct call to switch_mm() or activate_mm() with
147 * interrupts enabled and a new context.
148 */
149 switch_new_context(mm);
150} 90}
151 91
152#define init_new_context(tsk,mm) (__init_new_context(tsk,mm),0) 92/*
93 * It would be nice to return ASIDs back to the allocator, but unfortunately
94 * that introduces a race with a generation rollover where we could erroneously
95 * free an ASID allocated in a future generation. We could workaround this by
96 * freeing the ASID from the context of the dying mm (e.g. in arch_exit_mmap),
97 * but we'd then need to make sure that we didn't dirty any TLBs afterwards.
98 * Setting a reserved TTBR0 or EPD0 would work, but it all gets ugly when you
99 * take CPU migration into account.
100 */
153#define destroy_context(mm) do { } while(0) 101#define destroy_context(mm) do { } while(0)
102void check_and_switch_context(struct mm_struct *mm, unsigned int cpu);
154 103
155#define finish_arch_post_lock_switch \ 104#define init_new_context(tsk,mm) ({ atomic64_set(&(mm)->context.id, 0); 0; })
156 finish_arch_post_lock_switch
157static inline void finish_arch_post_lock_switch(void)
158{
159 if (test_and_clear_thread_flag(TIF_SWITCH_MM)) {
160 struct mm_struct *mm = current->mm;
161 unsigned long flags;
162
163 __new_context(mm);
164
165 local_irq_save(flags);
166 cpu_switch_mm(mm->pgd, mm);
167 local_irq_restore(flags);
168 }
169}
170 105
171/* 106/*
172 * This is called when "tsk" is about to enter lazy TLB mode. 107 * This is called when "tsk" is about to enter lazy TLB mode.
@@ -194,6 +129,9 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
194{ 129{
195 unsigned int cpu = smp_processor_id(); 130 unsigned int cpu = smp_processor_id();
196 131
132 if (prev == next)
133 return;
134
197 /* 135 /*
198 * init_mm.pgd does not contain any user mappings and it is always 136 * init_mm.pgd does not contain any user mappings and it is always
199 * active for kernel addresses in TTBR1. Just set the reserved TTBR0. 137 * active for kernel addresses in TTBR1. Just set the reserved TTBR0.
@@ -203,8 +141,7 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
203 return; 141 return;
204 } 142 }
205 143
206 if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next)) || prev != next) 144 check_and_switch_context(next, cpu);
207 check_and_switch_context(next, tsk);
208} 145}
209 146
210#define deactivate_mm(tsk,mm) do { } while (0) 147#define deactivate_mm(tsk,mm) do { } while (0)
diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
index 7d9c7e4a424b..9b2f5a9d019d 100644
--- a/arch/arm64/include/asm/page.h
+++ b/arch/arm64/include/asm/page.h
@@ -20,31 +20,22 @@
20#define __ASM_PAGE_H 20#define __ASM_PAGE_H
21 21
22/* PAGE_SHIFT determines the page size */ 22/* PAGE_SHIFT determines the page size */
23/* CONT_SHIFT determines the number of pages which can be tracked together */
23#ifdef CONFIG_ARM64_64K_PAGES 24#ifdef CONFIG_ARM64_64K_PAGES
24#define PAGE_SHIFT 16 25#define PAGE_SHIFT 16
26#define CONT_SHIFT 5
27#elif defined(CONFIG_ARM64_16K_PAGES)
28#define PAGE_SHIFT 14
29#define CONT_SHIFT 7
25#else 30#else
26#define PAGE_SHIFT 12 31#define PAGE_SHIFT 12
32#define CONT_SHIFT 4
27#endif 33#endif
28#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) 34#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
29#define PAGE_MASK (~(PAGE_SIZE-1)) 35#define PAGE_MASK (~(PAGE_SIZE-1))
30 36
31/* 37#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
32 * The idmap and swapper page tables need some space reserved in the kernel 38#define CONT_MASK (~(CONT_SIZE-1))
33 * image. Both require pgd, pud (4 levels only) and pmd tables to (section)
34 * map the kernel. With the 64K page configuration, swapper and idmap need to
35 * map to pte level. The swapper also maps the FDT (see __create_page_tables
36 * for more information). Note that the number of ID map translation levels
37 * could be increased on the fly if system RAM is out of reach for the default
38 * VA range, so 3 pages are reserved in all cases.
39 */
40#ifdef CONFIG_ARM64_64K_PAGES
41#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS)
42#else
43#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
44#endif
45
46#define SWAPPER_DIR_SIZE (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
47#define IDMAP_DIR_SIZE (3 * PAGE_SIZE)
48 39
49#ifndef __ASSEMBLY__ 40#ifndef __ASSEMBLY__
50 41
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 76420568d66a..c15053902942 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -27,6 +27,7 @@
27#define check_pgt_cache() do { } while (0) 27#define check_pgt_cache() do { } while (0)
28 28
29#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) 29#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO)
30#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
30 31
31#if CONFIG_PGTABLE_LEVELS > 2 32#if CONFIG_PGTABLE_LEVELS > 2
32 33
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index 24154b055835..d6739e836f7b 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -16,13 +16,46 @@
16#ifndef __ASM_PGTABLE_HWDEF_H 16#ifndef __ASM_PGTABLE_HWDEF_H
17#define __ASM_PGTABLE_HWDEF_H 17#define __ASM_PGTABLE_HWDEF_H
18 18
19/*
20 * Number of page-table levels required to address 'va_bits' wide
21 * address, without section mapping. We resolve the top (va_bits - PAGE_SHIFT)
22 * bits with (PAGE_SHIFT - 3) bits at each page table level. Hence:
23 *
24 * levels = DIV_ROUND_UP((va_bits - PAGE_SHIFT), (PAGE_SHIFT - 3))
25 *
26 * where DIV_ROUND_UP(n, d) => (((n) + (d) - 1) / (d))
27 *
28 * We cannot include linux/kernel.h which defines DIV_ROUND_UP here
29 * due to build issues. So we open code DIV_ROUND_UP here:
30 *
31 * ((((va_bits) - PAGE_SHIFT) + (PAGE_SHIFT - 3) - 1) / (PAGE_SHIFT - 3))
32 *
33 * which gets simplified as :
34 */
35#define ARM64_HW_PGTABLE_LEVELS(va_bits) (((va_bits) - 4) / (PAGE_SHIFT - 3))
36
37/*
38 * Size mapped by an entry at level n ( 0 <= n <= 3)
39 * We map (PAGE_SHIFT - 3) at all translation levels and PAGE_SHIFT bits
40 * in the final page. The maximum number of translation levels supported by
41 * the architecture is 4. Hence, starting at at level n, we have further
42 * ((4 - n) - 1) levels of translation excluding the offset within the page.
43 * So, the total number of bits mapped by an entry at level n is :
44 *
45 * ((4 - n) - 1) * (PAGE_SHIFT - 3) + PAGE_SHIFT
46 *
47 * Rearranging it a bit we get :
48 * (4 - n) * (PAGE_SHIFT - 3) + 3
49 */
50#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) ((PAGE_SHIFT - 3) * (4 - (n)) + 3)
51
19#define PTRS_PER_PTE (1 << (PAGE_SHIFT - 3)) 52#define PTRS_PER_PTE (1 << (PAGE_SHIFT - 3))
20 53
21/* 54/*
22 * PMD_SHIFT determines the size a level 2 page table entry can map. 55 * PMD_SHIFT determines the size a level 2 page table entry can map.
23 */ 56 */
24#if CONFIG_PGTABLE_LEVELS > 2 57#if CONFIG_PGTABLE_LEVELS > 2
25#define PMD_SHIFT ((PAGE_SHIFT - 3) * 2 + 3) 58#define PMD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(2)
26#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) 59#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT)
27#define PMD_MASK (~(PMD_SIZE-1)) 60#define PMD_MASK (~(PMD_SIZE-1))
28#define PTRS_PER_PMD PTRS_PER_PTE 61#define PTRS_PER_PMD PTRS_PER_PTE
@@ -32,7 +65,7 @@
32 * PUD_SHIFT determines the size a level 1 page table entry can map. 65 * PUD_SHIFT determines the size a level 1 page table entry can map.
33 */ 66 */
34#if CONFIG_PGTABLE_LEVELS > 3 67#if CONFIG_PGTABLE_LEVELS > 3
35#define PUD_SHIFT ((PAGE_SHIFT - 3) * 3 + 3) 68#define PUD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(1)
36#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) 69#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT)
37#define PUD_MASK (~(PUD_SIZE-1)) 70#define PUD_MASK (~(PUD_SIZE-1))
38#define PTRS_PER_PUD PTRS_PER_PTE 71#define PTRS_PER_PUD PTRS_PER_PTE
@@ -42,7 +75,7 @@
42 * PGDIR_SHIFT determines the size a top-level page table entry can map 75 * PGDIR_SHIFT determines the size a top-level page table entry can map
43 * (depending on the configuration, this level can be 0, 1 or 2). 76 * (depending on the configuration, this level can be 0, 1 or 2).
44 */ 77 */
45#define PGDIR_SHIFT ((PAGE_SHIFT - 3) * CONFIG_PGTABLE_LEVELS + 3) 78#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS)
46#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) 79#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
47#define PGDIR_MASK (~(PGDIR_SIZE-1)) 80#define PGDIR_MASK (~(PGDIR_SIZE-1))
48#define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) 81#define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT))
@@ -55,6 +88,13 @@
55#define SECTION_MASK (~(SECTION_SIZE-1)) 88#define SECTION_MASK (~(SECTION_SIZE-1))
56 89
57/* 90/*
91 * Contiguous page definitions.
92 */
93#define CONT_PTES (_AC(1, UL) << CONT_SHIFT)
94/* the the numerical offset of the PTE within a range of CONT_PTES */
95#define CONT_RANGE_OFFSET(addr) (((addr)>>PAGE_SHIFT)&(CONT_PTES-1))
96
97/*
58 * Hardware page table definitions. 98 * Hardware page table definitions.
59 * 99 *
60 * Level 1 descriptor (PUD). 100 * Level 1 descriptor (PUD).
@@ -83,6 +123,7 @@
83#define PMD_SECT_S (_AT(pmdval_t, 3) << 8) 123#define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
84#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10) 124#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10)
85#define PMD_SECT_NG (_AT(pmdval_t, 1) << 11) 125#define PMD_SECT_NG (_AT(pmdval_t, 1) << 11)
126#define PMD_SECT_CONT (_AT(pmdval_t, 1) << 52)
86#define PMD_SECT_PXN (_AT(pmdval_t, 1) << 53) 127#define PMD_SECT_PXN (_AT(pmdval_t, 1) << 53)
87#define PMD_SECT_UXN (_AT(pmdval_t, 1) << 54) 128#define PMD_SECT_UXN (_AT(pmdval_t, 1) << 54)
88 129
@@ -105,6 +146,7 @@
105#define PTE_AF (_AT(pteval_t, 1) << 10) /* Access Flag */ 146#define PTE_AF (_AT(pteval_t, 1) << 10) /* Access Flag */
106#define PTE_NG (_AT(pteval_t, 1) << 11) /* nG */ 147#define PTE_NG (_AT(pteval_t, 1) << 11) /* nG */
107#define PTE_DBM (_AT(pteval_t, 1) << 51) /* Dirty Bit Management */ 148#define PTE_DBM (_AT(pteval_t, 1) << 51) /* Dirty Bit Management */
149#define PTE_CONT (_AT(pteval_t, 1) << 52) /* Contiguous range */
108#define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */ 150#define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */
109#define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */ 151#define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */
110 152
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 26b066690593..7e074f93f383 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -41,7 +41,14 @@
41 * fixed mappings and modules 41 * fixed mappings and modules
42 */ 42 */
43#define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE) 43#define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE)
44#define VMALLOC_START (UL(0xffffffffffffffff) << VA_BITS) 44
45#ifndef CONFIG_KASAN
46#define VMALLOC_START (VA_START)
47#else
48#include <asm/kasan.h>
49#define VMALLOC_START (KASAN_SHADOW_END + SZ_64K)
50#endif
51
45#define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K) 52#define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
46 53
47#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K)) 54#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
@@ -60,8 +67,10 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
60#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) 67#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
61#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) 68#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
62 69
70#define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRnE))
63#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE)) 71#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE))
64#define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL_NC)) 72#define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL_NC))
73#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL_WT))
65#define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL)) 74#define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL))
66 75
67#define PROT_SECT_DEVICE_nGnRE (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE)) 76#define PROT_SECT_DEVICE_nGnRE (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE))
@@ -71,7 +80,10 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
71#define _PAGE_DEFAULT (PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL)) 80#define _PAGE_DEFAULT (PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
72 81
73#define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE) 82#define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE)
83#define PAGE_KERNEL_RO __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
84#define PAGE_KERNEL_ROX __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
74#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE) 85#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE)
86#define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT)
75 87
76#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP) 88#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP)
77#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP) 89#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
@@ -140,6 +152,7 @@ extern struct page *empty_zero_page;
140#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) 152#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))
141#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) 153#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))
142#define pte_exec(pte) (!(pte_val(pte) & PTE_UXN)) 154#define pte_exec(pte) (!(pte_val(pte) & PTE_UXN))
155#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
143 156
144#ifdef CONFIG_ARM64_HW_AFDBM 157#ifdef CONFIG_ARM64_HW_AFDBM
145#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY)) 158#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
@@ -202,6 +215,16 @@ static inline pte_t pte_mkspecial(pte_t pte)
202 return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); 215 return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
203} 216}
204 217
218static inline pte_t pte_mkcont(pte_t pte)
219{
220 return set_pte_bit(pte, __pgprot(PTE_CONT));
221}
222
223static inline pte_t pte_mknoncont(pte_t pte)
224{
225 return clear_pte_bit(pte, __pgprot(PTE_CONT));
226}
227
205static inline void set_pte(pte_t *ptep, pte_t pte) 228static inline void set_pte(pte_t *ptep, pte_t pte)
206{ 229{
207 *ptep = pte; 230 *ptep = pte;
@@ -646,14 +669,17 @@ static inline void update_mmu_cache(struct vm_area_struct *vma,
646 unsigned long addr, pte_t *ptep) 669 unsigned long addr, pte_t *ptep)
647{ 670{
648 /* 671 /*
649 * set_pte() does not have a DSB for user mappings, so make sure that 672 * We don't do anything here, so there's a very small chance of
650 * the page table write is visible. 673 * us retaking a user fault which we just fixed up. The alternative
674 * is doing a dsb(ishst), but that penalises the fastpath.
651 */ 675 */
652 dsb(ishst);
653} 676}
654 677
655#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) 678#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
656 679
680#define kc_vaddr_to_offset(v) ((v) & ~VA_START)
681#define kc_offset_to_vaddr(o) ((o) | VA_START)
682
657#endif /* !__ASSEMBLY__ */ 683#endif /* !__ASSEMBLY__ */
658 684
659#endif /* __ASM_PGTABLE_H */ 685#endif /* __ASM_PGTABLE_H */
diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h
deleted file mode 100644
index b7710a59672c..000000000000
--- a/arch/arm64/include/asm/pmu.h
+++ /dev/null
@@ -1,83 +0,0 @@
1/*
2 * Based on arch/arm/include/asm/pmu.h
3 *
4 * Copyright (C) 2009 picoChip Designs Ltd, Jamie Iles
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19#ifndef __ASM_PMU_H
20#define __ASM_PMU_H
21
22#ifdef CONFIG_HW_PERF_EVENTS
23
24/* The events for a given PMU register set. */
25struct pmu_hw_events {
26 /*
27 * The events that are active on the PMU for the given index.
28 */
29 struct perf_event **events;
30
31 /*
32 * A 1 bit for an index indicates that the counter is being used for
33 * an event. A 0 means that the counter can be used.
34 */
35 unsigned long *used_mask;
36
37 /*
38 * Hardware lock to serialize accesses to PMU registers. Needed for the
39 * read/modify/write sequences.
40 */
41 raw_spinlock_t pmu_lock;
42};
43
44struct arm_pmu {
45 struct pmu pmu;
46 cpumask_t active_irqs;
47 int *irq_affinity;
48 const char *name;
49 irqreturn_t (*handle_irq)(int irq_num, void *dev);
50 void (*enable)(struct hw_perf_event *evt, int idx);
51 void (*disable)(struct hw_perf_event *evt, int idx);
52 int (*get_event_idx)(struct pmu_hw_events *hw_events,
53 struct hw_perf_event *hwc);
54 int (*set_event_filter)(struct hw_perf_event *evt,
55 struct perf_event_attr *attr);
56 u32 (*read_counter)(int idx);
57 void (*write_counter)(int idx, u32 val);
58 void (*start)(void);
59 void (*stop)(void);
60 void (*reset)(void *);
61 int (*map_event)(struct perf_event *event);
62 int num_events;
63 atomic_t active_events;
64 struct mutex reserve_mutex;
65 u64 max_period;
66 struct platform_device *plat_device;
67 struct pmu_hw_events *(*get_hw_events)(void);
68};
69
70#define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
71
72int __init armpmu_register(struct arm_pmu *armpmu, char *name, int type);
73
74u64 armpmu_event_update(struct perf_event *event,
75 struct hw_perf_event *hwc,
76 int idx);
77
78int armpmu_event_set_period(struct perf_event *event,
79 struct hw_perf_event *hwc,
80 int idx);
81
82#endif /* CONFIG_HW_PERF_EVENTS */
83#endif /* __ASM_PMU_H */
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 98f32355dc97..4acb7ca94fcd 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -186,6 +186,6 @@ static inline void spin_lock_prefetch(const void *x)
186 186
187#endif 187#endif
188 188
189void cpu_enable_pan(void); 189void cpu_enable_pan(void *__unused);
190 190
191#endif /* __ASM_PROCESSOR_H */ 191#endif /* __ASM_PROCESSOR_H */
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index 536274ed292e..e9e5467e0bf4 100644
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -83,14 +83,14 @@
83#define compat_sp regs[13] 83#define compat_sp regs[13]
84#define compat_lr regs[14] 84#define compat_lr regs[14]
85#define compat_sp_hyp regs[15] 85#define compat_sp_hyp regs[15]
86#define compat_sp_irq regs[16] 86#define compat_lr_irq regs[16]
87#define compat_lr_irq regs[17] 87#define compat_sp_irq regs[17]
88#define compat_sp_svc regs[18] 88#define compat_lr_svc regs[18]
89#define compat_lr_svc regs[19] 89#define compat_sp_svc regs[19]
90#define compat_sp_abt regs[20] 90#define compat_lr_abt regs[20]
91#define compat_lr_abt regs[21] 91#define compat_sp_abt regs[21]
92#define compat_sp_und regs[22] 92#define compat_lr_und regs[22]
93#define compat_lr_und regs[23] 93#define compat_sp_und regs[23]
94#define compat_r8_fiq regs[24] 94#define compat_r8_fiq regs[24]
95#define compat_r9_fiq regs[25] 95#define compat_r9_fiq regs[25]
96#define compat_r10_fiq regs[26] 96#define compat_r10_fiq regs[26]
diff --git a/arch/arm64/include/asm/string.h b/arch/arm64/include/asm/string.h
index 64d2d4884a9d..2eb714c4639f 100644
--- a/arch/arm64/include/asm/string.h
+++ b/arch/arm64/include/asm/string.h
@@ -36,17 +36,33 @@ extern __kernel_size_t strnlen(const char *, __kernel_size_t);
36 36
37#define __HAVE_ARCH_MEMCPY 37#define __HAVE_ARCH_MEMCPY
38extern void *memcpy(void *, const void *, __kernel_size_t); 38extern void *memcpy(void *, const void *, __kernel_size_t);
39extern void *__memcpy(void *, const void *, __kernel_size_t);
39 40
40#define __HAVE_ARCH_MEMMOVE 41#define __HAVE_ARCH_MEMMOVE
41extern void *memmove(void *, const void *, __kernel_size_t); 42extern void *memmove(void *, const void *, __kernel_size_t);
43extern void *__memmove(void *, const void *, __kernel_size_t);
42 44
43#define __HAVE_ARCH_MEMCHR 45#define __HAVE_ARCH_MEMCHR
44extern void *memchr(const void *, int, __kernel_size_t); 46extern void *memchr(const void *, int, __kernel_size_t);
45 47
46#define __HAVE_ARCH_MEMSET 48#define __HAVE_ARCH_MEMSET
47extern void *memset(void *, int, __kernel_size_t); 49extern void *memset(void *, int, __kernel_size_t);
50extern void *__memset(void *, int, __kernel_size_t);
48 51
49#define __HAVE_ARCH_MEMCMP 52#define __HAVE_ARCH_MEMCMP
50extern int memcmp(const void *, const void *, size_t); 53extern int memcmp(const void *, const void *, size_t);
51 54
55
56#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
57
58/*
59 * For files that are not instrumented (e.g. mm/slub.c) we
60 * should use not instrumented version of mem* functions.
61 */
62
63#define memcpy(dst, src, len) __memcpy(dst, src, len)
64#define memmove(dst, src, len) __memmove(dst, src, len)
65#define memset(s, c, n) __memset(s, c, n)
66#endif
67
52#endif 68#endif
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index a7f3d4b2514d..d48ab5b41f52 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -22,9 +22,6 @@
22 22
23#include <asm/opcodes.h> 23#include <asm/opcodes.h>
24 24
25#define SCTLR_EL1_CP15BEN (0x1 << 5)
26#define SCTLR_EL1_SED (0x1 << 8)
27
28/* 25/*
29 * ARMv8 ARM reserves the following encoding for system registers: 26 * ARMv8 ARM reserves the following encoding for system registers:
30 * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", 27 * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview",
@@ -38,12 +35,162 @@
38#define sys_reg(op0, op1, crn, crm, op2) \ 35#define sys_reg(op0, op1, crn, crm, op2) \
39 ((((op0)&3)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) 36 ((((op0)&3)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5))
40 37
41#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) 38#define SYS_MIDR_EL1 sys_reg(3, 0, 0, 0, 0)
42#define SCTLR_EL1_SPAN (1 << 23) 39#define SYS_MPIDR_EL1 sys_reg(3, 0, 0, 0, 5)
40#define SYS_REVIDR_EL1 sys_reg(3, 0, 0, 0, 6)
41
42#define SYS_ID_PFR0_EL1 sys_reg(3, 0, 0, 1, 0)
43#define SYS_ID_PFR1_EL1 sys_reg(3, 0, 0, 1, 1)
44#define SYS_ID_DFR0_EL1 sys_reg(3, 0, 0, 1, 2)
45#define SYS_ID_MMFR0_EL1 sys_reg(3, 0, 0, 1, 4)
46#define SYS_ID_MMFR1_EL1 sys_reg(3, 0, 0, 1, 5)
47#define SYS_ID_MMFR2_EL1 sys_reg(3, 0, 0, 1, 6)
48#define SYS_ID_MMFR3_EL1 sys_reg(3, 0, 0, 1, 7)
49
50#define SYS_ID_ISAR0_EL1 sys_reg(3, 0, 0, 2, 0)
51#define SYS_ID_ISAR1_EL1 sys_reg(3, 0, 0, 2, 1)
52#define SYS_ID_ISAR2_EL1 sys_reg(3, 0, 0, 2, 2)
53#define SYS_ID_ISAR3_EL1 sys_reg(3, 0, 0, 2, 3)
54#define SYS_ID_ISAR4_EL1 sys_reg(3, 0, 0, 2, 4)
55#define SYS_ID_ISAR5_EL1 sys_reg(3, 0, 0, 2, 5)
56#define SYS_ID_MMFR4_EL1 sys_reg(3, 0, 0, 2, 6)
57
58#define SYS_MVFR0_EL1 sys_reg(3, 0, 0, 3, 0)
59#define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1)
60#define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2)
61
62#define SYS_ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 4, 0)
63#define SYS_ID_AA64PFR1_EL1 sys_reg(3, 0, 0, 4, 1)
64
65#define SYS_ID_AA64DFR0_EL1 sys_reg(3, 0, 0, 5, 0)
66#define SYS_ID_AA64DFR1_EL1 sys_reg(3, 0, 0, 5, 1)
67
68#define SYS_ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0)
69#define SYS_ID_AA64ISAR1_EL1 sys_reg(3, 0, 0, 6, 1)
70
71#define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0)
72#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1)
73
74#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
75#define SYS_CTR_EL0 sys_reg(3, 3, 0, 0, 1)
76#define SYS_DCZID_EL0 sys_reg(3, 3, 0, 0, 7)
77
78#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4)
43 79
44#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\ 80#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\
45 (!!x)<<8 | 0x1f) 81 (!!x)<<8 | 0x1f)
46 82
83/* SCTLR_EL1 */
84#define SCTLR_EL1_CP15BEN (0x1 << 5)
85#define SCTLR_EL1_SED (0x1 << 8)
86#define SCTLR_EL1_SPAN (0x1 << 23)
87
88
89/* id_aa64isar0 */
90#define ID_AA64ISAR0_RDM_SHIFT 28
91#define ID_AA64ISAR0_ATOMICS_SHIFT 20
92#define ID_AA64ISAR0_CRC32_SHIFT 16
93#define ID_AA64ISAR0_SHA2_SHIFT 12
94#define ID_AA64ISAR0_SHA1_SHIFT 8
95#define ID_AA64ISAR0_AES_SHIFT 4
96
97/* id_aa64pfr0 */
98#define ID_AA64PFR0_GIC_SHIFT 24
99#define ID_AA64PFR0_ASIMD_SHIFT 20
100#define ID_AA64PFR0_FP_SHIFT 16
101#define ID_AA64PFR0_EL3_SHIFT 12
102#define ID_AA64PFR0_EL2_SHIFT 8
103#define ID_AA64PFR0_EL1_SHIFT 4
104#define ID_AA64PFR0_EL0_SHIFT 0
105
106#define ID_AA64PFR0_FP_NI 0xf
107#define ID_AA64PFR0_FP_SUPPORTED 0x0
108#define ID_AA64PFR0_ASIMD_NI 0xf
109#define ID_AA64PFR0_ASIMD_SUPPORTED 0x0
110#define ID_AA64PFR0_EL1_64BIT_ONLY 0x1
111#define ID_AA64PFR0_EL0_64BIT_ONLY 0x1
112
113/* id_aa64mmfr0 */
114#define ID_AA64MMFR0_TGRAN4_SHIFT 28
115#define ID_AA64MMFR0_TGRAN64_SHIFT 24
116#define ID_AA64MMFR0_TGRAN16_SHIFT 20
117#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16
118#define ID_AA64MMFR0_SNSMEM_SHIFT 12
119#define ID_AA64MMFR0_BIGENDEL_SHIFT 8
120#define ID_AA64MMFR0_ASID_SHIFT 4
121#define ID_AA64MMFR0_PARANGE_SHIFT 0
122
123#define ID_AA64MMFR0_TGRAN4_NI 0xf
124#define ID_AA64MMFR0_TGRAN4_SUPPORTED 0x0
125#define ID_AA64MMFR0_TGRAN64_NI 0xf
126#define ID_AA64MMFR0_TGRAN64_SUPPORTED 0x0
127#define ID_AA64MMFR0_TGRAN16_NI 0x0
128#define ID_AA64MMFR0_TGRAN16_SUPPORTED 0x1
129
130/* id_aa64mmfr1 */
131#define ID_AA64MMFR1_PAN_SHIFT 20
132#define ID_AA64MMFR1_LOR_SHIFT 16
133#define ID_AA64MMFR1_HPD_SHIFT 12
134#define ID_AA64MMFR1_VHE_SHIFT 8
135#define ID_AA64MMFR1_VMIDBITS_SHIFT 4
136#define ID_AA64MMFR1_HADBS_SHIFT 0
137
138/* id_aa64dfr0 */
139#define ID_AA64DFR0_CTX_CMPS_SHIFT 28
140#define ID_AA64DFR0_WRPS_SHIFT 20
141#define ID_AA64DFR0_BRPS_SHIFT 12
142#define ID_AA64DFR0_PMUVER_SHIFT 8
143#define ID_AA64DFR0_TRACEVER_SHIFT 4
144#define ID_AA64DFR0_DEBUGVER_SHIFT 0
145
146#define ID_ISAR5_RDM_SHIFT 24
147#define ID_ISAR5_CRC32_SHIFT 16
148#define ID_ISAR5_SHA2_SHIFT 12
149#define ID_ISAR5_SHA1_SHIFT 8
150#define ID_ISAR5_AES_SHIFT 4
151#define ID_ISAR5_SEVL_SHIFT 0
152
153#define MVFR0_FPROUND_SHIFT 28
154#define MVFR0_FPSHVEC_SHIFT 24
155#define MVFR0_FPSQRT_SHIFT 20
156#define MVFR0_FPDIVIDE_SHIFT 16
157#define MVFR0_FPTRAP_SHIFT 12
158#define MVFR0_FPDP_SHIFT 8
159#define MVFR0_FPSP_SHIFT 4
160#define MVFR0_SIMD_SHIFT 0
161
162#define MVFR1_SIMDFMAC_SHIFT 28
163#define MVFR1_FPHP_SHIFT 24
164#define MVFR1_SIMDHP_SHIFT 20
165#define MVFR1_SIMDSP_SHIFT 16
166#define MVFR1_SIMDINT_SHIFT 12
167#define MVFR1_SIMDLS_SHIFT 8
168#define MVFR1_FPDNAN_SHIFT 4
169#define MVFR1_FPFTZ_SHIFT 0
170
171
172#define ID_AA64MMFR0_TGRAN4_SHIFT 28
173#define ID_AA64MMFR0_TGRAN64_SHIFT 24
174#define ID_AA64MMFR0_TGRAN16_SHIFT 20
175
176#define ID_AA64MMFR0_TGRAN4_NI 0xf
177#define ID_AA64MMFR0_TGRAN4_SUPPORTED 0x0
178#define ID_AA64MMFR0_TGRAN64_NI 0xf
179#define ID_AA64MMFR0_TGRAN64_SUPPORTED 0x0
180#define ID_AA64MMFR0_TGRAN16_NI 0x0
181#define ID_AA64MMFR0_TGRAN16_SUPPORTED 0x1
182
183#if defined(CONFIG_ARM64_4K_PAGES)
184#define ID_AA64MMFR0_TGRAN_SHIFT ID_AA64MMFR0_TGRAN4_SHIFT
185#define ID_AA64MMFR0_TGRAN_SUPPORTED ID_AA64MMFR0_TGRAN4_SUPPORTED
186#elif defined(CONFIG_ARM64_16K_PAGES)
187#define ID_AA64MMFR0_TGRAN_SHIFT ID_AA64MMFR0_TGRAN16_SHIFT
188#define ID_AA64MMFR0_TGRAN_SUPPORTED ID_AA64MMFR0_TGRAN16_SUPPORTED
189#elif defined(CONFIG_ARM64_64K_PAGES)
190#define ID_AA64MMFR0_TGRAN_SHIFT ID_AA64MMFR0_TGRAN64_SHIFT
191#define ID_AA64MMFR0_TGRAN_SUPPORTED ID_AA64MMFR0_TGRAN64_SUPPORTED
192#endif
193
47#ifdef __ASSEMBLY__ 194#ifdef __ASSEMBLY__
48 195
49 .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 196 .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index dcd06d18a42a..90c7ff233735 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -23,8 +23,10 @@
23 23
24#include <linux/compiler.h> 24#include <linux/compiler.h>
25 25
26#ifndef CONFIG_ARM64_64K_PAGES 26#ifdef CONFIG_ARM64_4K_PAGES
27#define THREAD_SIZE_ORDER 2 27#define THREAD_SIZE_ORDER 2
28#elif defined(CONFIG_ARM64_16K_PAGES)
29#define THREAD_SIZE_ORDER 0
28#endif 30#endif
29 31
30#define THREAD_SIZE 16384 32#define THREAD_SIZE 16384
@@ -111,7 +113,6 @@ static inline struct thread_info *current_thread_info(void)
111#define TIF_RESTORE_SIGMASK 20 113#define TIF_RESTORE_SIGMASK 20
112#define TIF_SINGLESTEP 21 114#define TIF_SINGLESTEP 21
113#define TIF_32BIT 22 /* 32bit process */ 115#define TIF_32BIT 22 /* 32bit process */
114#define TIF_SWITCH_MM 23 /* deferred switch_mm */
115 116
116#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) 117#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
117#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) 118#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index d6e6b6660380..ffdaea7954bb 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -37,17 +37,21 @@ static inline void __tlb_remove_table(void *_table)
37 37
38static inline void tlb_flush(struct mmu_gather *tlb) 38static inline void tlb_flush(struct mmu_gather *tlb)
39{ 39{
40 if (tlb->fullmm) { 40 struct vm_area_struct vma = { .vm_mm = tlb->mm, };
41 flush_tlb_mm(tlb->mm); 41
42 } else { 42 /*
43 struct vm_area_struct vma = { .vm_mm = tlb->mm, }; 43 * The ASID allocator will either invalidate the ASID or mark
44 /* 44 * it as used.
45 * The intermediate page table levels are already handled by 45 */
46 * the __(pte|pmd|pud)_free_tlb() functions, so last level 46 if (tlb->fullmm)
47 * TLBI is sufficient here. 47 return;
48 */ 48
49 __flush_tlb_range(&vma, tlb->start, tlb->end, true); 49 /*
50 } 50 * The intermediate page table levels are already handled by
51 * the __(pte|pmd|pud)_free_tlb() functions, so last level
52 * TLBI is sufficient here.
53 */
54 __flush_tlb_range(&vma, tlb->start, tlb->end, true);
51} 55}
52 56
53static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 57static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 7bd2da021658..b460ae28e346 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -63,6 +63,14 @@
63 * only require the D-TLB to be invalidated. 63 * only require the D-TLB to be invalidated.
64 * - kaddr - Kernel virtual memory address 64 * - kaddr - Kernel virtual memory address
65 */ 65 */
66static inline void local_flush_tlb_all(void)
67{
68 dsb(nshst);
69 asm("tlbi vmalle1");
70 dsb(nsh);
71 isb();
72}
73
66static inline void flush_tlb_all(void) 74static inline void flush_tlb_all(void)
67{ 75{
68 dsb(ishst); 76 dsb(ishst);
@@ -73,7 +81,7 @@ static inline void flush_tlb_all(void)
73 81
74static inline void flush_tlb_mm(struct mm_struct *mm) 82static inline void flush_tlb_mm(struct mm_struct *mm)
75{ 83{
76 unsigned long asid = (unsigned long)ASID(mm) << 48; 84 unsigned long asid = ASID(mm) << 48;
77 85
78 dsb(ishst); 86 dsb(ishst);
79 asm("tlbi aside1is, %0" : : "r" (asid)); 87 asm("tlbi aside1is, %0" : : "r" (asid));
@@ -83,8 +91,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
83static inline void flush_tlb_page(struct vm_area_struct *vma, 91static inline void flush_tlb_page(struct vm_area_struct *vma,
84 unsigned long uaddr) 92 unsigned long uaddr)
85{ 93{
86 unsigned long addr = uaddr >> 12 | 94 unsigned long addr = uaddr >> 12 | (ASID(vma->vm_mm) << 48);
87 ((unsigned long)ASID(vma->vm_mm) << 48);
88 95
89 dsb(ishst); 96 dsb(ishst);
90 asm("tlbi vale1is, %0" : : "r" (addr)); 97 asm("tlbi vale1is, %0" : : "r" (addr));
@@ -101,7 +108,7 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
101 unsigned long start, unsigned long end, 108 unsigned long start, unsigned long end,
102 bool last_level) 109 bool last_level)
103{ 110{
104 unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48; 111 unsigned long asid = ASID(vma->vm_mm) << 48;
105 unsigned long addr; 112 unsigned long addr;
106 113
107 if ((end - start) > MAX_TLB_RANGE) { 114 if ((end - start) > MAX_TLB_RANGE) {
@@ -154,9 +161,8 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
154static inline void __flush_tlb_pgtable(struct mm_struct *mm, 161static inline void __flush_tlb_pgtable(struct mm_struct *mm,
155 unsigned long uaddr) 162 unsigned long uaddr)
156{ 163{
157 unsigned long addr = uaddr >> 12 | ((unsigned long)ASID(mm) << 48); 164 unsigned long addr = uaddr >> 12 | (ASID(mm) << 48);
158 165
159 dsb(ishst);
160 asm("tlbi vae1is, %0" : : "r" (addr)); 166 asm("tlbi vae1is, %0" : : "r" (addr));
161 dsb(ish); 167 dsb(ish);
162} 168}
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 0cd7b5947dfc..2d4ca4bb0dd3 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -32,7 +32,7 @@
32 32
33#ifndef __ASSEMBLY__ 33#ifndef __ASSEMBLY__
34#include <linux/psci.h> 34#include <linux/psci.h>
35#include <asm/types.h> 35#include <linux/types.h>
36#include <asm/ptrace.h> 36#include <asm/ptrace.h>
37 37
38#define __KVM_HAVE_GUEST_DEBUG 38#define __KVM_HAVE_GUEST_DEBUG
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 22dc9bc781be..474691f8b13a 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -4,7 +4,6 @@
4 4
5CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) 5CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET)
6AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) 6AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
7CFLAGS_efi-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
8CFLAGS_armv8_deprecated.o := -I$(src) 7CFLAGS_armv8_deprecated.o := -I$(src)
9 8
10CFLAGS_REMOVE_ftrace.o = -pg 9CFLAGS_REMOVE_ftrace.o = -pg
@@ -20,6 +19,12 @@ arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
20 cpufeature.o alternative.o cacheinfo.o \ 19 cpufeature.o alternative.o cacheinfo.o \
21 smp.o smp_spin_table.o topology.o 20 smp.o smp_spin_table.o topology.o
22 21
22extra-$(CONFIG_EFI) := efi-entry.o
23
24OBJCOPYFLAGS := --prefix-symbols=__efistub_
25$(obj)/%.stub.o: $(obj)/%.o FORCE
26 $(call if_changed,objcopy)
27
23arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ 28arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
24 sys_compat.o entry32.o \ 29 sys_compat.o entry32.o \
25 ../../arm/kernel/opcodes.o 30 ../../arm/kernel/opcodes.o
@@ -32,7 +37,7 @@ arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o
32arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o 37arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o
33arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o 38arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
34arm64-obj-$(CONFIG_KGDB) += kgdb.o 39arm64-obj-$(CONFIG_KGDB) += kgdb.o
35arm64-obj-$(CONFIG_EFI) += efi.o efi-stub.o efi-entry.o 40arm64-obj-$(CONFIG_EFI) += efi.o efi-entry.stub.o
36arm64-obj-$(CONFIG_PCI) += pci.o 41arm64-obj-$(CONFIG_PCI) += pci.o
37arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o 42arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o
38arm64-obj-$(CONFIG_ACPI) += acpi.o 43arm64-obj-$(CONFIG_ACPI) += acpi.o
@@ -40,7 +45,7 @@ arm64-obj-$(CONFIG_ACPI) += acpi.o
40obj-y += $(arm64-obj-y) vdso/ 45obj-y += $(arm64-obj-y) vdso/
41obj-m += $(arm64-obj-m) 46obj-m += $(arm64-obj-m)
42head-y := head.o 47head-y := head.o
43extra-y := $(head-y) vmlinux.lds 48extra-y += $(head-y) vmlinux.lds
44 49
45# vDSO - this must be built first to generate the symbol offsets 50# vDSO - this must be built first to generate the symbol offsets
46$(call objectify,$(arm64-obj-y)): $(obj)/vdso/vdso-offsets.h 51$(call objectify,$(arm64-obj-y)): $(obj)/vdso/vdso-offsets.h
diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c
index 19de7537e7d3..d1ce8e2f98b9 100644
--- a/arch/arm64/kernel/acpi.c
+++ b/arch/arm64/kernel/acpi.c
@@ -29,6 +29,11 @@
29#include <asm/cpu_ops.h> 29#include <asm/cpu_ops.h>
30#include <asm/smp_plat.h> 30#include <asm/smp_plat.h>
31 31
32#ifdef CONFIG_ACPI_APEI
33# include <linux/efi.h>
34# include <asm/pgtable.h>
35#endif
36
32int acpi_noirq = 1; /* skip ACPI IRQ initialization */ 37int acpi_noirq = 1; /* skip ACPI IRQ initialization */
33int acpi_disabled = 1; 38int acpi_disabled = 1;
34EXPORT_SYMBOL(acpi_disabled); 39EXPORT_SYMBOL(acpi_disabled);
@@ -206,27 +211,26 @@ void __init acpi_boot_table_init(void)
206 } 211 }
207} 212}
208 213
209void __init acpi_gic_init(void) 214#ifdef CONFIG_ACPI_APEI
215pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr)
210{ 216{
211 struct acpi_table_header *table; 217 /*
212 acpi_status status; 218 * According to "Table 8 Map: EFI memory types to AArch64 memory
213 acpi_size tbl_size; 219 * types" of UEFI 2.5 section 2.3.6.1, each EFI memory type is
214 int err; 220 * mapped to a corresponding MAIR attribute encoding.
215 221 * The EFI memory attribute advises all possible capabilities
216 if (acpi_disabled) 222 * of a memory region. We use the most efficient capability.
217 return; 223 */
218
219 status = acpi_get_table_with_size(ACPI_SIG_MADT, 0, &table, &tbl_size);
220 if (ACPI_FAILURE(status)) {
221 const char *msg = acpi_format_exception(status);
222
223 pr_err("Failed to get MADT table, %s\n", msg);
224 return;
225 }
226 224
227 err = gic_v2_acpi_init(table); 225 u64 attr;
228 if (err)
229 pr_err("Failed to initialize GIC IRQ controller");
230 226
231 early_acpi_os_unmap_memory((char *)table, tbl_size); 227 attr = efi_mem_attributes(addr);
228 if (attr & EFI_MEMORY_WB)
229 return PAGE_KERNEL;
230 if (attr & EFI_MEMORY_WT)
231 return __pgprot(PROT_NORMAL_WT);
232 if (attr & EFI_MEMORY_WC)
233 return __pgprot(PROT_NORMAL_NC);
234 return __pgprot(PROT_DEVICE_nGnRnE);
232} 235}
236#endif
diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
index a85843ddbde8..3b6d8cc9dfe0 100644
--- a/arch/arm64/kernel/arm64ksyms.c
+++ b/arch/arm64/kernel/arm64ksyms.c
@@ -51,6 +51,9 @@ EXPORT_SYMBOL(strnlen);
51EXPORT_SYMBOL(memset); 51EXPORT_SYMBOL(memset);
52EXPORT_SYMBOL(memcpy); 52EXPORT_SYMBOL(memcpy);
53EXPORT_SYMBOL(memmove); 53EXPORT_SYMBOL(memmove);
54EXPORT_SYMBOL(__memset);
55EXPORT_SYMBOL(__memcpy);
56EXPORT_SYMBOL(__memmove);
54EXPORT_SYMBOL(memchr); 57EXPORT_SYMBOL(memchr);
55EXPORT_SYMBOL(memcmp); 58EXPORT_SYMBOL(memcmp);
56 59
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index bcee7abac68e..937f5e58a4d3 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -284,21 +284,23 @@ static void register_insn_emulation_sysctl(struct ctl_table *table)
284 __asm__ __volatile__( \ 284 __asm__ __volatile__( \
285 ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ 285 ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
286 CONFIG_ARM64_PAN) \ 286 CONFIG_ARM64_PAN) \
287 " mov %w2, %w1\n" \ 287 "0: ldxr"B" %w2, [%3]\n" \
288 "0: ldxr"B" %w1, [%3]\n" \ 288 "1: stxr"B" %w0, %w1, [%3]\n" \
289 "1: stxr"B" %w0, %w2, [%3]\n" \
290 " cbz %w0, 2f\n" \ 289 " cbz %w0, 2f\n" \
291 " mov %w0, %w4\n" \ 290 " mov %w0, %w4\n" \
291 " b 3f\n" \
292 "2:\n" \ 292 "2:\n" \
293 " mov %w1, %w2\n" \
294 "3:\n" \
293 " .pushsection .fixup,\"ax\"\n" \ 295 " .pushsection .fixup,\"ax\"\n" \
294 " .align 2\n" \ 296 " .align 2\n" \
295 "3: mov %w0, %w5\n" \ 297 "4: mov %w0, %w5\n" \
296 " b 2b\n" \ 298 " b 3b\n" \
297 " .popsection" \ 299 " .popsection" \
298 " .pushsection __ex_table,\"a\"\n" \ 300 " .pushsection __ex_table,\"a\"\n" \
299 " .align 3\n" \ 301 " .align 3\n" \
300 " .quad 0b, 3b\n" \ 302 " .quad 0b, 4b\n" \
301 " .quad 1b, 3b\n" \ 303 " .quad 1b, 4b\n" \
302 " .popsection\n" \ 304 " .popsection\n" \
303 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ 305 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
304 CONFIG_ARM64_PAN) \ 306 CONFIG_ARM64_PAN) \
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 8d89cf8dae55..25de8b244961 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -60,7 +60,7 @@ int main(void)
60 DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno)); 60 DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno));
61 DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); 61 DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs));
62 BLANK(); 62 BLANK();
63 DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id)); 63 DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id.counter));
64 BLANK(); 64 BLANK();
65 DEFINE(VMA_VM_MM, offsetof(struct vm_area_struct, vm_mm)); 65 DEFINE(VMA_VM_MM, offsetof(struct vm_area_struct, vm_mm));
66 DEFINE(VMA_VM_FLAGS, offsetof(struct vm_area_struct, vm_flags)); 66 DEFINE(VMA_VM_FLAGS, offsetof(struct vm_area_struct, vm_flags));
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 6ffd91438560..feb6b4efa641 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -23,6 +23,7 @@
23 23
24#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) 24#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
25#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) 25#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
26#define MIDR_THUNDERX MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
26 27
27#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \ 28#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
28 MIDR_ARCHITECTURE_MASK) 29 MIDR_ARCHITECTURE_MASK)
@@ -74,6 +75,15 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
74 (1 << MIDR_VARIANT_SHIFT) | 2), 75 (1 << MIDR_VARIANT_SHIFT) | 2),
75 }, 76 },
76#endif 77#endif
78#ifdef CONFIG_ARM64_ERRATUM_834220
79 {
80 /* Cortex-A57 r0p0 - r1p2 */
81 .desc = "ARM erratum 834220",
82 .capability = ARM64_WORKAROUND_834220,
83 MIDR_RANGE(MIDR_CORTEX_A57, 0x00,
84 (1 << MIDR_VARIANT_SHIFT) | 2),
85 },
86#endif
77#ifdef CONFIG_ARM64_ERRATUM_845719 87#ifdef CONFIG_ARM64_ERRATUM_845719
78 { 88 {
79 /* Cortex-A53 r0p[01234] */ 89 /* Cortex-A53 r0p[01234] */
@@ -82,11 +92,19 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
82 MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04), 92 MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04),
83 }, 93 },
84#endif 94#endif
95#ifdef CONFIG_CAVIUM_ERRATUM_23154
96 {
97 /* Cavium ThunderX, pass 1.x */
98 .desc = "Cavium erratum 23154",
99 .capability = ARM64_WORKAROUND_CAVIUM_23154,
100 MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01),
101 },
102#endif
85 { 103 {
86 } 104 }
87}; 105};
88 106
89void check_local_cpu_errata(void) 107void check_local_cpu_errata(void)
90{ 108{
91 check_cpu_capabilities(arm64_errata, "enabling workaround for"); 109 update_cpu_capabilities(arm64_errata, "enabling workaround for");
92} 110}
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 3c9aed32f70b..c8cf89223b5a 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -16,12 +16,569 @@
16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */ 17 */
18 18
19#define pr_fmt(fmt) "alternatives: " fmt 19#define pr_fmt(fmt) "CPU features: " fmt
20 20
21#include <linux/bsearch.h>
22#include <linux/sort.h>
21#include <linux/types.h> 23#include <linux/types.h>
22#include <asm/cpu.h> 24#include <asm/cpu.h>
23#include <asm/cpufeature.h> 25#include <asm/cpufeature.h>
26#include <asm/cpu_ops.h>
24#include <asm/processor.h> 27#include <asm/processor.h>
28#include <asm/sysreg.h>
29
30unsigned long elf_hwcap __read_mostly;
31EXPORT_SYMBOL_GPL(elf_hwcap);
32
33#ifdef CONFIG_COMPAT
34#define COMPAT_ELF_HWCAP_DEFAULT \
35 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
36 COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
37 COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
38 COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
39 COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
40 COMPAT_HWCAP_LPAE)
41unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
42unsigned int compat_elf_hwcap2 __read_mostly;
43#endif
44
45DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
46
47#define ARM64_FTR_BITS(STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
48 { \
49 .strict = STRICT, \
50 .type = TYPE, \
51 .shift = SHIFT, \
52 .width = WIDTH, \
53 .safe_val = SAFE_VAL, \
54 }
55
56#define ARM64_FTR_END \
57 { \
58 .width = 0, \
59 }
60
61static struct arm64_ftr_bits ftr_id_aa64isar0[] = {
62 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
63 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
64 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 24, 4, 0),
65 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
66 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
67 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
68 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0),
69 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0),
70 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* RAZ */
71 ARM64_FTR_END,
72};
73
74static struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
75 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
76 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 4, 0),
77 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0),
78 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
79 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
80 /* Linux doesn't care about the EL3 */
81 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, ID_AA64PFR0_EL3_SHIFT, 4, 0),
82 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL2_SHIFT, 4, 0),
83 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
84 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
85 ARM64_FTR_END,
86};
87
88static struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
89 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
90 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
91 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
92 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
93 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
94 /* Linux shouldn't care about secure memory */
95 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
96 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
97 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
98 /*
99 * Differing PARange is fine as long as all peripherals and memory are mapped
100 * within the minimum PARange of all CPUs
101 */
102 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
103 ARM64_FTR_END,
104};
105
106static struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
107 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
108 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
109 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
110 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
111 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
112 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
113 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
114 ARM64_FTR_END,
115};
116
117static struct arm64_ftr_bits ftr_ctr[] = {
118 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RAO */
119 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 3, 0),
120 ARM64_FTR_BITS(FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0), /* CWG */
121 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0), /* ERG */
122 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1), /* DminLine */
123 /*
124 * Linux can handle differing I-cache policies. Userspace JITs will
125 * make use of *minLine
126 */
127 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, 14, 2, 0), /* L1Ip */
128 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 10, 0), /* RAZ */
129 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* IminLine */
130 ARM64_FTR_END,
131};
132
133static struct arm64_ftr_bits ftr_id_mmfr0[] = {
134 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 4, 0), /* InnerShr */
135 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 24, 4, 0), /* FCSE */
136 ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0), /* AuxReg */
137 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 16, 4, 0), /* TCM */
138 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 12, 4, 0), /* ShareLvl */
139 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 4, 0), /* OuterShr */
140 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* PMSA */
141 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* VMSA */
142 ARM64_FTR_END,
143};
144
145static struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
146 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
147 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
148 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
149 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
150 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
151 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_TRACEVER_SHIFT, 4, 0),
152 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
153 ARM64_FTR_END,
154};
155
156static struct arm64_ftr_bits ftr_mvfr2[] = {
157 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 24, 0), /* RAZ */
158 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* FPMisc */
159 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* SIMDMisc */
160 ARM64_FTR_END,
161};
162
163static struct arm64_ftr_bits ftr_dczid[] = {
164 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 5, 27, 0), /* RAZ */
165 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 1, 1), /* DZP */
166 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* BS */
167 ARM64_FTR_END,
168};
169
170
171static struct arm64_ftr_bits ftr_id_isar5[] = {
172 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_RDM_SHIFT, 4, 0),
173 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 20, 4, 0), /* RAZ */
174 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_CRC32_SHIFT, 4, 0),
175 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA2_SHIFT, 4, 0),
176 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA1_SHIFT, 4, 0),
177 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_AES_SHIFT, 4, 0),
178 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SEVL_SHIFT, 4, 0),
179 ARM64_FTR_END,
180};
181
182static struct arm64_ftr_bits ftr_id_mmfr4[] = {
183 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 24, 0), /* RAZ */
184 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* ac2 */
185 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* RAZ */
186 ARM64_FTR_END,
187};
188
189static struct arm64_ftr_bits ftr_id_pfr0[] = {
190 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 16, 16, 0), /* RAZ */
191 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 12, 4, 0), /* State3 */
192 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 4, 0), /* State2 */
193 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* State1 */
194 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* State0 */
195 ARM64_FTR_END,
196};
197
198/*
199 * Common ftr bits for a 32bit register with all hidden, strict
200 * attributes, with 4bit feature fields and a default safe value of
201 * 0. Covers the following 32bit registers:
202 * id_isar[0-4], id_mmfr[1-3], id_pfr1, mvfr[0-1]
203 */
204static struct arm64_ftr_bits ftr_generic_32bits[] = {
205 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
206 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
207 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
208 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
209 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
210 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
211 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
212 ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
213 ARM64_FTR_END,
214};
215
216static struct arm64_ftr_bits ftr_generic[] = {
217 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 64, 0),
218 ARM64_FTR_END,
219};
220
221static struct arm64_ftr_bits ftr_generic32[] = {
222 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 32, 0),
223 ARM64_FTR_END,
224};
225
226static struct arm64_ftr_bits ftr_aa64raz[] = {
227 ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 64, 0),
228 ARM64_FTR_END,
229};
230
231#define ARM64_FTR_REG(id, table) \
232 { \
233 .sys_id = id, \
234 .name = #id, \
235 .ftr_bits = &((table)[0]), \
236 }
237
238static struct arm64_ftr_reg arm64_ftr_regs[] = {
239
240 /* Op1 = 0, CRn = 0, CRm = 1 */
241 ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
242 ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_generic_32bits),
243 ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_generic_32bits),
244 ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
245 ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
246 ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
247 ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
248
249 /* Op1 = 0, CRn = 0, CRm = 2 */
250 ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_generic_32bits),
251 ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
252 ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
253 ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
254 ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_generic_32bits),
255 ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
256 ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
257
258 /* Op1 = 0, CRn = 0, CRm = 3 */
259 ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits),
260 ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits),
261 ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
262
263 /* Op1 = 0, CRn = 0, CRm = 4 */
264 ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
265 ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_aa64raz),
266
267 /* Op1 = 0, CRn = 0, CRm = 5 */
268 ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
269 ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_generic),
270
271 /* Op1 = 0, CRn = 0, CRm = 6 */
272 ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
273 ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_aa64raz),
274
275 /* Op1 = 0, CRn = 0, CRm = 7 */
276 ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
277 ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
278
279 /* Op1 = 3, CRn = 0, CRm = 0 */
280 ARM64_FTR_REG(SYS_CTR_EL0, ftr_ctr),
281 ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
282
283 /* Op1 = 3, CRn = 14, CRm = 0 */
284 ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_generic32),
285};
286
287static int search_cmp_ftr_reg(const void *id, const void *regp)
288{
289 return (int)(unsigned long)id - (int)((const struct arm64_ftr_reg *)regp)->sys_id;
290}
291
292/*
293 * get_arm64_ftr_reg - Lookup a feature register entry using its
294 * sys_reg() encoding. With the array arm64_ftr_regs sorted in the
295 * ascending order of sys_id , we use binary search to find a matching
296 * entry.
297 *
298 * returns - Upon success, matching ftr_reg entry for id.
299 * - NULL on failure. It is upto the caller to decide
300 * the impact of a failure.
301 */
302static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
303{
304 return bsearch((const void *)(unsigned long)sys_id,
305 arm64_ftr_regs,
306 ARRAY_SIZE(arm64_ftr_regs),
307 sizeof(arm64_ftr_regs[0]),
308 search_cmp_ftr_reg);
309}
310
311static u64 arm64_ftr_set_value(struct arm64_ftr_bits *ftrp, s64 reg, s64 ftr_val)
312{
313 u64 mask = arm64_ftr_mask(ftrp);
314
315 reg &= ~mask;
316 reg |= (ftr_val << ftrp->shift) & mask;
317 return reg;
318}
319
320static s64 arm64_ftr_safe_value(struct arm64_ftr_bits *ftrp, s64 new, s64 cur)
321{
322 s64 ret = 0;
323
324 switch (ftrp->type) {
325 case FTR_EXACT:
326 ret = ftrp->safe_val;
327 break;
328 case FTR_LOWER_SAFE:
329 ret = new < cur ? new : cur;
330 break;
331 case FTR_HIGHER_SAFE:
332 ret = new > cur ? new : cur;
333 break;
334 default:
335 BUG();
336 }
337
338 return ret;
339}
340
341static int __init sort_cmp_ftr_regs(const void *a, const void *b)
342{
343 return ((const struct arm64_ftr_reg *)a)->sys_id -
344 ((const struct arm64_ftr_reg *)b)->sys_id;
345}
346
347static void __init swap_ftr_regs(void *a, void *b, int size)
348{
349 struct arm64_ftr_reg tmp = *(struct arm64_ftr_reg *)a;
350 *(struct arm64_ftr_reg *)a = *(struct arm64_ftr_reg *)b;
351 *(struct arm64_ftr_reg *)b = tmp;
352}
353
354static void __init sort_ftr_regs(void)
355{
356 /* Keep the array sorted so that we can do the binary search */
357 sort(arm64_ftr_regs,
358 ARRAY_SIZE(arm64_ftr_regs),
359 sizeof(arm64_ftr_regs[0]),
360 sort_cmp_ftr_regs,
361 swap_ftr_regs);
362}
363
364/*
365 * Initialise the CPU feature register from Boot CPU values.
366 * Also initiliases the strict_mask for the register.
367 */
368static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
369{
370 u64 val = 0;
371 u64 strict_mask = ~0x0ULL;
372 struct arm64_ftr_bits *ftrp;
373 struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
374
375 BUG_ON(!reg);
376
377 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
378 s64 ftr_new = arm64_ftr_value(ftrp, new);
379
380 val = arm64_ftr_set_value(ftrp, val, ftr_new);
381 if (!ftrp->strict)
382 strict_mask &= ~arm64_ftr_mask(ftrp);
383 }
384 reg->sys_val = val;
385 reg->strict_mask = strict_mask;
386}
387
388void __init init_cpu_features(struct cpuinfo_arm64 *info)
389{
390 /* Before we start using the tables, make sure it is sorted */
391 sort_ftr_regs();
392
393 init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
394 init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
395 init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
396 init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
397 init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
398 init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
399 init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
400 init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
401 init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
402 init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
403 init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
404 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
405 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
406 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
407 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
408 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
409 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
410 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
411 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
412 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
413 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
414 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
415 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
416 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
417 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
418 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
419 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
420}
421
422static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
423{
424 struct arm64_ftr_bits *ftrp;
425
426 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
427 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
428 s64 ftr_new = arm64_ftr_value(ftrp, new);
429
430 if (ftr_cur == ftr_new)
431 continue;
432 /* Find a safe value */
433 ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
434 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
435 }
436
437}
438
439static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
440{
441 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
442
443 BUG_ON(!regp);
444 update_cpu_ftr_reg(regp, val);
445 if ((boot & regp->strict_mask) == (val & regp->strict_mask))
446 return 0;
447 pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
448 regp->name, boot, cpu, val);
449 return 1;
450}
451
452/*
453 * Update system wide CPU feature registers with the values from a
454 * non-boot CPU. Also performs SANITY checks to make sure that there
455 * aren't any insane variations from that of the boot CPU.
456 */
457void update_cpu_features(int cpu,
458 struct cpuinfo_arm64 *info,
459 struct cpuinfo_arm64 *boot)
460{
461 int taint = 0;
462
463 /*
464 * The kernel can handle differing I-cache policies, but otherwise
465 * caches should look identical. Userspace JITs will make use of
466 * *minLine.
467 */
468 taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
469 info->reg_ctr, boot->reg_ctr);
470
471 /*
472 * Userspace may perform DC ZVA instructions. Mismatched block sizes
473 * could result in too much or too little memory being zeroed if a
474 * process is preempted and migrated between CPUs.
475 */
476 taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
477 info->reg_dczid, boot->reg_dczid);
478
479 /* If different, timekeeping will be broken (especially with KVM) */
480 taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
481 info->reg_cntfrq, boot->reg_cntfrq);
482
483 /*
484 * The kernel uses self-hosted debug features and expects CPUs to
485 * support identical debug features. We presently need CTX_CMPs, WRPs,
486 * and BRPs to be identical.
487 * ID_AA64DFR1 is currently RES0.
488 */
489 taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
490 info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
491 taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
492 info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
493 /*
494 * Even in big.LITTLE, processors should be identical instruction-set
495 * wise.
496 */
497 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
498 info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
499 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
500 info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
501
502 /*
503 * Differing PARange support is fine as long as all peripherals and
504 * memory are mapped within the minimum PARange of all CPUs.
505 * Linux should not care about secure memory.
506 */
507 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
508 info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
509 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
510 info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
511
512 /*
513 * EL3 is not our concern.
514 * ID_AA64PFR1 is currently RES0.
515 */
516 taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
517 info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
518 taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
519 info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
520
521 /*
522 * If we have AArch32, we care about 32-bit features for compat. These
523 * registers should be RES0 otherwise.
524 */
525 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
526 info->reg_id_dfr0, boot->reg_id_dfr0);
527 taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
528 info->reg_id_isar0, boot->reg_id_isar0);
529 taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
530 info->reg_id_isar1, boot->reg_id_isar1);
531 taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
532 info->reg_id_isar2, boot->reg_id_isar2);
533 taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
534 info->reg_id_isar3, boot->reg_id_isar3);
535 taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
536 info->reg_id_isar4, boot->reg_id_isar4);
537 taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
538 info->reg_id_isar5, boot->reg_id_isar5);
539
540 /*
541 * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
542 * ACTLR formats could differ across CPUs and therefore would have to
543 * be trapped for virtualization anyway.
544 */
545 taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
546 info->reg_id_mmfr0, boot->reg_id_mmfr0);
547 taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
548 info->reg_id_mmfr1, boot->reg_id_mmfr1);
549 taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
550 info->reg_id_mmfr2, boot->reg_id_mmfr2);
551 taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
552 info->reg_id_mmfr3, boot->reg_id_mmfr3);
553 taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
554 info->reg_id_pfr0, boot->reg_id_pfr0);
555 taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
556 info->reg_id_pfr1, boot->reg_id_pfr1);
557 taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
558 info->reg_mvfr0, boot->reg_mvfr0);
559 taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
560 info->reg_mvfr1, boot->reg_mvfr1);
561 taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
562 info->reg_mvfr2, boot->reg_mvfr2);
563
564 /*
565 * Mismatched CPU features are a recipe for disaster. Don't even
566 * pretend to support them.
567 */
568 WARN_TAINT_ONCE(taint, TAINT_CPU_OUT_OF_SPEC,
569 "Unsupported CPU feature variation.\n");
570}
571
572u64 read_system_reg(u32 id)
573{
574 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
575
576 /* We shouldn't get a request for an unsupported register */
577 BUG_ON(!regp);
578 return regp->sys_val;
579}
580
581#include <linux/irqchip/arm-gic-v3.h>
25 582
26static bool 583static bool
27feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) 584feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
@@ -31,34 +588,46 @@ feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
31 return val >= entry->min_field_value; 588 return val >= entry->min_field_value;
32} 589}
33 590
34#define __ID_FEAT_CHK(reg) \ 591static bool
35static bool __maybe_unused \ 592has_cpuid_feature(const struct arm64_cpu_capabilities *entry)
36has_##reg##_feature(const struct arm64_cpu_capabilities *entry) \ 593{
37{ \ 594 u64 val;
38 u64 val; \ 595
39 \ 596 val = read_system_reg(entry->sys_reg);
40 val = read_cpuid(reg##_el1); \ 597 return feature_matches(val, entry);
41 return feature_matches(val, entry); \
42} 598}
43 599
44__ID_FEAT_CHK(id_aa64pfr0); 600static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry)
45__ID_FEAT_CHK(id_aa64mmfr1); 601{
46__ID_FEAT_CHK(id_aa64isar0); 602 bool has_sre;
603
604 if (!has_cpuid_feature(entry))
605 return false;
606
607 has_sre = gic_enable_sre();
608 if (!has_sre)
609 pr_warn_once("%s present but disabled by higher exception level\n",
610 entry->desc);
611
612 return has_sre;
613}
47 614
48static const struct arm64_cpu_capabilities arm64_features[] = { 615static const struct arm64_cpu_capabilities arm64_features[] = {
49 { 616 {
50 .desc = "GIC system register CPU interface", 617 .desc = "GIC system register CPU interface",
51 .capability = ARM64_HAS_SYSREG_GIC_CPUIF, 618 .capability = ARM64_HAS_SYSREG_GIC_CPUIF,
52 .matches = has_id_aa64pfr0_feature, 619 .matches = has_useable_gicv3_cpuif,
53 .field_pos = 24, 620 .sys_reg = SYS_ID_AA64PFR0_EL1,
621 .field_pos = ID_AA64PFR0_GIC_SHIFT,
54 .min_field_value = 1, 622 .min_field_value = 1,
55 }, 623 },
56#ifdef CONFIG_ARM64_PAN 624#ifdef CONFIG_ARM64_PAN
57 { 625 {
58 .desc = "Privileged Access Never", 626 .desc = "Privileged Access Never",
59 .capability = ARM64_HAS_PAN, 627 .capability = ARM64_HAS_PAN,
60 .matches = has_id_aa64mmfr1_feature, 628 .matches = has_cpuid_feature,
61 .field_pos = 20, 629 .sys_reg = SYS_ID_AA64MMFR1_EL1,
630 .field_pos = ID_AA64MMFR1_PAN_SHIFT,
62 .min_field_value = 1, 631 .min_field_value = 1,
63 .enable = cpu_enable_pan, 632 .enable = cpu_enable_pan,
64 }, 633 },
@@ -67,15 +636,101 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
67 { 636 {
68 .desc = "LSE atomic instructions", 637 .desc = "LSE atomic instructions",
69 .capability = ARM64_HAS_LSE_ATOMICS, 638 .capability = ARM64_HAS_LSE_ATOMICS,
70 .matches = has_id_aa64isar0_feature, 639 .matches = has_cpuid_feature,
71 .field_pos = 20, 640 .sys_reg = SYS_ID_AA64ISAR0_EL1,
641 .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
72 .min_field_value = 2, 642 .min_field_value = 2,
73 }, 643 },
74#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */ 644#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
75 {}, 645 {},
76}; 646};
77 647
78void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, 648#define HWCAP_CAP(reg, field, min_value, type, cap) \
649 { \
650 .desc = #cap, \
651 .matches = has_cpuid_feature, \
652 .sys_reg = reg, \
653 .field_pos = field, \
654 .min_field_value = min_value, \
655 .hwcap_type = type, \
656 .hwcap = cap, \
657 }
658
659static const struct arm64_cpu_capabilities arm64_hwcaps[] = {
660 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, 2, CAP_HWCAP, HWCAP_PMULL),
661 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, 1, CAP_HWCAP, HWCAP_AES),
662 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, 1, CAP_HWCAP, HWCAP_SHA1),
663 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, 1, CAP_HWCAP, HWCAP_SHA2),
664 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, 1, CAP_HWCAP, HWCAP_CRC32),
665 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, 2, CAP_HWCAP, HWCAP_ATOMICS),
666 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 0, CAP_HWCAP, HWCAP_FP),
667 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 0, CAP_HWCAP, HWCAP_ASIMD),
668#ifdef CONFIG_COMPAT
669 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
670 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
671 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
672 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
673 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
674#endif
675 {},
676};
677
678static void cap_set_hwcap(const struct arm64_cpu_capabilities *cap)
679{
680 switch (cap->hwcap_type) {
681 case CAP_HWCAP:
682 elf_hwcap |= cap->hwcap;
683 break;
684#ifdef CONFIG_COMPAT
685 case CAP_COMPAT_HWCAP:
686 compat_elf_hwcap |= (u32)cap->hwcap;
687 break;
688 case CAP_COMPAT_HWCAP2:
689 compat_elf_hwcap2 |= (u32)cap->hwcap;
690 break;
691#endif
692 default:
693 WARN_ON(1);
694 break;
695 }
696}
697
698/* Check if we have a particular HWCAP enabled */
699static bool __maybe_unused cpus_have_hwcap(const struct arm64_cpu_capabilities *cap)
700{
701 bool rc;
702
703 switch (cap->hwcap_type) {
704 case CAP_HWCAP:
705 rc = (elf_hwcap & cap->hwcap) != 0;
706 break;
707#ifdef CONFIG_COMPAT
708 case CAP_COMPAT_HWCAP:
709 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
710 break;
711 case CAP_COMPAT_HWCAP2:
712 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
713 break;
714#endif
715 default:
716 WARN_ON(1);
717 rc = false;
718 }
719
720 return rc;
721}
722
723static void setup_cpu_hwcaps(void)
724{
725 int i;
726 const struct arm64_cpu_capabilities *hwcaps = arm64_hwcaps;
727
728 for (i = 0; hwcaps[i].desc; i++)
729 if (hwcaps[i].matches(&hwcaps[i]))
730 cap_set_hwcap(&hwcaps[i]);
731}
732
733void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
79 const char *info) 734 const char *info)
80{ 735{
81 int i; 736 int i;
@@ -88,15 +743,178 @@ void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
88 pr_info("%s %s\n", info, caps[i].desc); 743 pr_info("%s %s\n", info, caps[i].desc);
89 cpus_set_cap(caps[i].capability); 744 cpus_set_cap(caps[i].capability);
90 } 745 }
746}
747
748/*
749 * Run through the enabled capabilities and enable() it on all active
750 * CPUs
751 */
752static void enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
753{
754 int i;
755
756 for (i = 0; caps[i].desc; i++)
757 if (caps[i].enable && cpus_have_cap(caps[i].capability))
758 on_each_cpu(caps[i].enable, NULL, true);
759}
760
761#ifdef CONFIG_HOTPLUG_CPU
762
763/*
764 * Flag to indicate if we have computed the system wide
765 * capabilities based on the boot time active CPUs. This
766 * will be used to determine if a new booting CPU should
767 * go through the verification process to make sure that it
768 * supports the system capabilities, without using a hotplug
769 * notifier.
770 */
771static bool sys_caps_initialised;
772
773static inline void set_sys_caps_initialised(void)
774{
775 sys_caps_initialised = true;
776}
777
778/*
779 * __raw_read_system_reg() - Used by a STARTING cpu before cpuinfo is populated.
780 */
781static u64 __raw_read_system_reg(u32 sys_id)
782{
783 switch (sys_id) {
784 case SYS_ID_PFR0_EL1: return (u64)read_cpuid(ID_PFR0_EL1);
785 case SYS_ID_PFR1_EL1: return (u64)read_cpuid(ID_PFR1_EL1);
786 case SYS_ID_DFR0_EL1: return (u64)read_cpuid(ID_DFR0_EL1);
787 case SYS_ID_MMFR0_EL1: return (u64)read_cpuid(ID_MMFR0_EL1);
788 case SYS_ID_MMFR1_EL1: return (u64)read_cpuid(ID_MMFR1_EL1);
789 case SYS_ID_MMFR2_EL1: return (u64)read_cpuid(ID_MMFR2_EL1);
790 case SYS_ID_MMFR3_EL1: return (u64)read_cpuid(ID_MMFR3_EL1);
791 case SYS_ID_ISAR0_EL1: return (u64)read_cpuid(ID_ISAR0_EL1);
792 case SYS_ID_ISAR1_EL1: return (u64)read_cpuid(ID_ISAR1_EL1);
793 case SYS_ID_ISAR2_EL1: return (u64)read_cpuid(ID_ISAR2_EL1);
794 case SYS_ID_ISAR3_EL1: return (u64)read_cpuid(ID_ISAR3_EL1);
795 case SYS_ID_ISAR4_EL1: return (u64)read_cpuid(ID_ISAR4_EL1);
796 case SYS_ID_ISAR5_EL1: return (u64)read_cpuid(ID_ISAR4_EL1);
797 case SYS_MVFR0_EL1: return (u64)read_cpuid(MVFR0_EL1);
798 case SYS_MVFR1_EL1: return (u64)read_cpuid(MVFR1_EL1);
799 case SYS_MVFR2_EL1: return (u64)read_cpuid(MVFR2_EL1);
800
801 case SYS_ID_AA64PFR0_EL1: return (u64)read_cpuid(ID_AA64PFR0_EL1);
802 case SYS_ID_AA64PFR1_EL1: return (u64)read_cpuid(ID_AA64PFR0_EL1);
803 case SYS_ID_AA64DFR0_EL1: return (u64)read_cpuid(ID_AA64DFR0_EL1);
804 case SYS_ID_AA64DFR1_EL1: return (u64)read_cpuid(ID_AA64DFR0_EL1);
805 case SYS_ID_AA64MMFR0_EL1: return (u64)read_cpuid(ID_AA64MMFR0_EL1);
806 case SYS_ID_AA64MMFR1_EL1: return (u64)read_cpuid(ID_AA64MMFR1_EL1);
807 case SYS_ID_AA64ISAR0_EL1: return (u64)read_cpuid(ID_AA64ISAR0_EL1);
808 case SYS_ID_AA64ISAR1_EL1: return (u64)read_cpuid(ID_AA64ISAR1_EL1);
91 809
92 /* second pass allows enable() to consider interacting capabilities */ 810 case SYS_CNTFRQ_EL0: return (u64)read_cpuid(CNTFRQ_EL0);
811 case SYS_CTR_EL0: return (u64)read_cpuid(CTR_EL0);
812 case SYS_DCZID_EL0: return (u64)read_cpuid(DCZID_EL0);
813 default:
814 BUG();
815 return 0;
816 }
817}
818
819/*
820 * Park the CPU which doesn't have the capability as advertised
821 * by the system.
822 */
823static void fail_incapable_cpu(char *cap_type,
824 const struct arm64_cpu_capabilities *cap)
825{
826 int cpu = smp_processor_id();
827
828 pr_crit("CPU%d: missing %s : %s\n", cpu, cap_type, cap->desc);
829 /* Mark this CPU absent */
830 set_cpu_present(cpu, 0);
831
832 /* Check if we can park ourselves */
833 if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die)
834 cpu_ops[cpu]->cpu_die(cpu);
835 asm(
836 "1: wfe\n"
837 " wfi\n"
838 " b 1b");
839}
840
841/*
842 * Run through the enabled system capabilities and enable() it on this CPU.
843 * The capabilities were decided based on the available CPUs at the boot time.
844 * Any new CPU should match the system wide status of the capability. If the
845 * new CPU doesn't have a capability which the system now has enabled, we
846 * cannot do anything to fix it up and could cause unexpected failures. So
847 * we park the CPU.
848 */
849void verify_local_cpu_capabilities(void)
850{
851 int i;
852 const struct arm64_cpu_capabilities *caps;
853
854 /*
855 * If we haven't computed the system capabilities, there is nothing
856 * to verify.
857 */
858 if (!sys_caps_initialised)
859 return;
860
861 caps = arm64_features;
93 for (i = 0; caps[i].desc; i++) { 862 for (i = 0; caps[i].desc; i++) {
94 if (cpus_have_cap(caps[i].capability) && caps[i].enable) 863 if (!cpus_have_cap(caps[i].capability) || !caps[i].sys_reg)
95 caps[i].enable(); 864 continue;
865 /*
866 * If the new CPU misses an advertised feature, we cannot proceed
867 * further, park the cpu.
868 */
869 if (!feature_matches(__raw_read_system_reg(caps[i].sys_reg), &caps[i]))
870 fail_incapable_cpu("arm64_features", &caps[i]);
871 if (caps[i].enable)
872 caps[i].enable(NULL);
96 } 873 }
874
875 for (i = 0, caps = arm64_hwcaps; caps[i].desc; i++) {
876 if (!cpus_have_hwcap(&caps[i]))
877 continue;
878 if (!feature_matches(__raw_read_system_reg(caps[i].sys_reg), &caps[i]))
879 fail_incapable_cpu("arm64_hwcaps", &caps[i]);
880 }
881}
882
883#else /* !CONFIG_HOTPLUG_CPU */
884
885static inline void set_sys_caps_initialised(void)
886{
887}
888
889#endif /* CONFIG_HOTPLUG_CPU */
890
891static void setup_feature_capabilities(void)
892{
893 update_cpu_capabilities(arm64_features, "detected feature:");
894 enable_cpu_capabilities(arm64_features);
97} 895}
98 896
99void check_local_cpu_features(void) 897void __init setup_cpu_features(void)
100{ 898{
101 check_cpu_capabilities(arm64_features, "detected feature:"); 899 u32 cwg;
900 int cls;
901
902 /* Set the CPU feature capabilies */
903 setup_feature_capabilities();
904 setup_cpu_hwcaps();
905
906 /* Advertise that we have computed the system capabilities */
907 set_sys_caps_initialised();
908
909 /*
910 * Check for sane CTR_EL0.CWG value.
911 */
912 cwg = cache_type_cwg();
913 cls = cache_line_size();
914 if (!cwg)
915 pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n",
916 cls);
917 if (L1_CACHE_BYTES < cls)
918 pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n",
919 L1_CACHE_BYTES, cls);
102} 920}
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 75d5a867e7fb..212ae6361d8b 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -24,9 +24,13 @@
24#include <linux/bug.h> 24#include <linux/bug.h>
25#include <linux/init.h> 25#include <linux/init.h>
26#include <linux/kernel.h> 26#include <linux/kernel.h>
27#include <linux/personality.h>
27#include <linux/preempt.h> 28#include <linux/preempt.h>
28#include <linux/printk.h> 29#include <linux/printk.h>
30#include <linux/seq_file.h>
31#include <linux/sched.h>
29#include <linux/smp.h> 32#include <linux/smp.h>
33#include <linux/delay.h>
30 34
31/* 35/*
32 * In case the boot CPU is hotpluggable, we record its initial state and 36 * In case the boot CPU is hotpluggable, we record its initial state and
@@ -35,7 +39,6 @@
35 */ 39 */
36DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data); 40DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data);
37static struct cpuinfo_arm64 boot_cpu_data; 41static struct cpuinfo_arm64 boot_cpu_data;
38static bool mixed_endian_el0 = true;
39 42
40static char *icache_policy_str[] = { 43static char *icache_policy_str[] = {
41 [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN", 44 [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN",
@@ -46,157 +49,152 @@ static char *icache_policy_str[] = {
46 49
47unsigned long __icache_flags; 50unsigned long __icache_flags;
48 51
49static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) 52static const char *const hwcap_str[] = {
53 "fp",
54 "asimd",
55 "evtstrm",
56 "aes",
57 "pmull",
58 "sha1",
59 "sha2",
60 "crc32",
61 "atomics",
62 NULL
63};
64
65#ifdef CONFIG_COMPAT
66static const char *const compat_hwcap_str[] = {
67 "swp",
68 "half",
69 "thumb",
70 "26bit",
71 "fastmult",
72 "fpa",
73 "vfp",
74 "edsp",
75 "java",
76 "iwmmxt",
77 "crunch",
78 "thumbee",
79 "neon",
80 "vfpv3",
81 "vfpv3d16",
82 "tls",
83 "vfpv4",
84 "idiva",
85 "idivt",
86 "vfpd32",
87 "lpae",
88 "evtstrm"
89};
90
91static const char *const compat_hwcap2_str[] = {
92 "aes",
93 "pmull",
94 "sha1",
95 "sha2",
96 "crc32",
97 NULL
98};
99#endif /* CONFIG_COMPAT */
100
101static int c_show(struct seq_file *m, void *v)
50{ 102{
51 unsigned int cpu = smp_processor_id(); 103 int i, j;
52 u32 l1ip = CTR_L1IP(info->reg_ctr); 104
105 for_each_online_cpu(i) {
106 struct cpuinfo_arm64 *cpuinfo = &per_cpu(cpu_data, i);
107 u32 midr = cpuinfo->reg_midr;
53 108
54 if (l1ip != ICACHE_POLICY_PIPT) {
55 /* 109 /*
56 * VIPT caches are non-aliasing if the VA always equals the PA 110 * glibc reads /proc/cpuinfo to determine the number of
57 * in all bit positions that are covered by the index. This is 111 * online processors, looking for lines beginning with
58 * the case if the size of a way (# of sets * line size) does 112 * "processor". Give glibc what it expects.
59 * not exceed PAGE_SIZE.
60 */ 113 */
61 u32 waysize = icache_get_numsets() * icache_get_linesize(); 114 seq_printf(m, "processor\t: %d\n", i);
62 115
63 if (l1ip != ICACHE_POLICY_VIPT || waysize > PAGE_SIZE) 116 seq_printf(m, "BogoMIPS\t: %lu.%02lu\n",
64 set_bit(ICACHEF_ALIASING, &__icache_flags); 117 loops_per_jiffy / (500000UL/HZ),
118 loops_per_jiffy / (5000UL/HZ) % 100);
119
120 /*
121 * Dump out the common processor features in a single line.
122 * Userspace should read the hwcaps with getauxval(AT_HWCAP)
123 * rather than attempting to parse this, but there's a body of
124 * software which does already (at least for 32-bit).
125 */
126 seq_puts(m, "Features\t:");
127 if (personality(current->personality) == PER_LINUX32) {
128#ifdef CONFIG_COMPAT
129 for (j = 0; compat_hwcap_str[j]; j++)
130 if (compat_elf_hwcap & (1 << j))
131 seq_printf(m, " %s", compat_hwcap_str[j]);
132
133 for (j = 0; compat_hwcap2_str[j]; j++)
134 if (compat_elf_hwcap2 & (1 << j))
135 seq_printf(m, " %s", compat_hwcap2_str[j]);
136#endif /* CONFIG_COMPAT */
137 } else {
138 for (j = 0; hwcap_str[j]; j++)
139 if (elf_hwcap & (1 << j))
140 seq_printf(m, " %s", hwcap_str[j]);
141 }
142 seq_puts(m, "\n");
143
144 seq_printf(m, "CPU implementer\t: 0x%02x\n",
145 MIDR_IMPLEMENTOR(midr));
146 seq_printf(m, "CPU architecture: 8\n");
147 seq_printf(m, "CPU variant\t: 0x%x\n", MIDR_VARIANT(midr));
148 seq_printf(m, "CPU part\t: 0x%03x\n", MIDR_PARTNUM(midr));
149 seq_printf(m, "CPU revision\t: %d\n\n", MIDR_REVISION(midr));
65 } 150 }
66 if (l1ip == ICACHE_POLICY_AIVIVT)
67 set_bit(ICACHEF_AIVIVT, &__icache_flags);
68 151
69 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); 152 return 0;
70} 153}
71 154
72bool cpu_supports_mixed_endian_el0(void) 155static void *c_start(struct seq_file *m, loff_t *pos)
73{ 156{
74 return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); 157 return *pos < 1 ? (void *)1 : NULL;
75} 158}
76 159
77bool system_supports_mixed_endian_el0(void) 160static void *c_next(struct seq_file *m, void *v, loff_t *pos)
78{ 161{
79 return mixed_endian_el0; 162 ++*pos;
163 return NULL;
80} 164}
81 165
82static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info) 166static void c_stop(struct seq_file *m, void *v)
83{ 167{
84 mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0);
85} 168}
86 169
87static void update_cpu_features(struct cpuinfo_arm64 *info) 170const struct seq_operations cpuinfo_op = {
88{ 171 .start = c_start,
89 update_mixed_endian_el0_support(info); 172 .next = c_next,
90} 173 .stop = c_stop,
174 .show = c_show
175};
91 176
92static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu) 177static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
93{ 178{
94 if ((boot & mask) == (cur & mask)) 179 unsigned int cpu = smp_processor_id();
95 return 0; 180 u32 l1ip = CTR_L1IP(info->reg_ctr);
96
97 pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016lx, CPU%d: %#016lx\n",
98 name, (unsigned long)boot, cpu, (unsigned long)cur);
99
100 return 1;
101}
102 181
103#define CHECK_MASK(field, mask, boot, cur, cpu) \ 182 if (l1ip != ICACHE_POLICY_PIPT) {
104 check_reg_mask(#field, mask, (boot)->reg_ ## field, (cur)->reg_ ## field, cpu) 183 /*
184 * VIPT caches are non-aliasing if the VA always equals the PA
185 * in all bit positions that are covered by the index. This is
186 * the case if the size of a way (# of sets * line size) does
187 * not exceed PAGE_SIZE.
188 */
189 u32 waysize = icache_get_numsets() * icache_get_linesize();
105 190
106#define CHECK(field, boot, cur, cpu) \ 191 if (l1ip != ICACHE_POLICY_VIPT || waysize > PAGE_SIZE)
107 CHECK_MASK(field, ~0ULL, boot, cur, cpu) 192 set_bit(ICACHEF_ALIASING, &__icache_flags);
193 }
194 if (l1ip == ICACHE_POLICY_AIVIVT)
195 set_bit(ICACHEF_AIVIVT, &__icache_flags);
108 196
109/* 197 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
110 * Verify that CPUs don't have unexpected differences that will cause problems.
111 */
112static void cpuinfo_sanity_check(struct cpuinfo_arm64 *cur)
113{
114 unsigned int cpu = smp_processor_id();
115 struct cpuinfo_arm64 *boot = &boot_cpu_data;
116 unsigned int diff = 0;
117
118 /*
119 * The kernel can handle differing I-cache policies, but otherwise
120 * caches should look identical. Userspace JITs will make use of
121 * *minLine.
122 */
123 diff |= CHECK_MASK(ctr, 0xffff3fff, boot, cur, cpu);
124
125 /*
126 * Userspace may perform DC ZVA instructions. Mismatched block sizes
127 * could result in too much or too little memory being zeroed if a
128 * process is preempted and migrated between CPUs.
129 */
130 diff |= CHECK(dczid, boot, cur, cpu);
131
132 /* If different, timekeeping will be broken (especially with KVM) */
133 diff |= CHECK(cntfrq, boot, cur, cpu);
134
135 /*
136 * The kernel uses self-hosted debug features and expects CPUs to
137 * support identical debug features. We presently need CTX_CMPs, WRPs,
138 * and BRPs to be identical.
139 * ID_AA64DFR1 is currently RES0.
140 */
141 diff |= CHECK(id_aa64dfr0, boot, cur, cpu);
142 diff |= CHECK(id_aa64dfr1, boot, cur, cpu);
143
144 /*
145 * Even in big.LITTLE, processors should be identical instruction-set
146 * wise.
147 */
148 diff |= CHECK(id_aa64isar0, boot, cur, cpu);
149 diff |= CHECK(id_aa64isar1, boot, cur, cpu);
150
151 /*
152 * Differing PARange support is fine as long as all peripherals and
153 * memory are mapped within the minimum PARange of all CPUs.
154 * Linux should not care about secure memory.
155 * ID_AA64MMFR1 is currently RES0.
156 */
157 diff |= CHECK_MASK(id_aa64mmfr0, 0xffffffffffff0ff0, boot, cur, cpu);
158 diff |= CHECK(id_aa64mmfr1, boot, cur, cpu);
159
160 /*
161 * EL3 is not our concern.
162 * ID_AA64PFR1 is currently RES0.
163 */
164 diff |= CHECK_MASK(id_aa64pfr0, 0xffffffffffff0fff, boot, cur, cpu);
165 diff |= CHECK(id_aa64pfr1, boot, cur, cpu);
166
167 /*
168 * If we have AArch32, we care about 32-bit features for compat. These
169 * registers should be RES0 otherwise.
170 */
171 diff |= CHECK(id_dfr0, boot, cur, cpu);
172 diff |= CHECK(id_isar0, boot, cur, cpu);
173 diff |= CHECK(id_isar1, boot, cur, cpu);
174 diff |= CHECK(id_isar2, boot, cur, cpu);
175 diff |= CHECK(id_isar3, boot, cur, cpu);
176 diff |= CHECK(id_isar4, boot, cur, cpu);
177 diff |= CHECK(id_isar5, boot, cur, cpu);
178 /*
179 * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
180 * ACTLR formats could differ across CPUs and therefore would have to
181 * be trapped for virtualization anyway.
182 */
183 diff |= CHECK_MASK(id_mmfr0, 0xff0fffff, boot, cur, cpu);
184 diff |= CHECK(id_mmfr1, boot, cur, cpu);
185 diff |= CHECK(id_mmfr2, boot, cur, cpu);
186 diff |= CHECK(id_mmfr3, boot, cur, cpu);
187 diff |= CHECK(id_pfr0, boot, cur, cpu);
188 diff |= CHECK(id_pfr1, boot, cur, cpu);
189
190 diff |= CHECK(mvfr0, boot, cur, cpu);
191 diff |= CHECK(mvfr1, boot, cur, cpu);
192 diff |= CHECK(mvfr2, boot, cur, cpu);
193
194 /*
195 * Mismatched CPU features are a recipe for disaster. Don't even
196 * pretend to support them.
197 */
198 WARN_TAINT_ONCE(diff, TAINT_CPU_OUT_OF_SPEC,
199 "Unsupported CPU feature variation.\n");
200} 198}
201 199
202static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) 200static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
@@ -236,15 +234,13 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
236 cpuinfo_detect_icache_policy(info); 234 cpuinfo_detect_icache_policy(info);
237 235
238 check_local_cpu_errata(); 236 check_local_cpu_errata();
239 check_local_cpu_features();
240 update_cpu_features(info);
241} 237}
242 238
243void cpuinfo_store_cpu(void) 239void cpuinfo_store_cpu(void)
244{ 240{
245 struct cpuinfo_arm64 *info = this_cpu_ptr(&cpu_data); 241 struct cpuinfo_arm64 *info = this_cpu_ptr(&cpu_data);
246 __cpuinfo_store_cpu(info); 242 __cpuinfo_store_cpu(info);
247 cpuinfo_sanity_check(info); 243 update_cpu_features(smp_processor_id(), info, &boot_cpu_data);
248} 244}
249 245
250void __init cpuinfo_store_boot_cpu(void) 246void __init cpuinfo_store_boot_cpu(void)
@@ -253,4 +249,5 @@ void __init cpuinfo_store_boot_cpu(void)
253 __cpuinfo_store_cpu(info); 249 __cpuinfo_store_cpu(info);
254 250
255 boot_cpu_data = *info; 251 boot_cpu_data = *info;
252 init_cpu_features(&boot_cpu_data);
256} 253}
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index 253021ef2769..8aee3aeec3e6 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -26,14 +26,16 @@
26#include <linux/stat.h> 26#include <linux/stat.h>
27#include <linux/uaccess.h> 27#include <linux/uaccess.h>
28 28
29#include <asm/debug-monitors.h> 29#include <asm/cpufeature.h>
30#include <asm/cputype.h> 30#include <asm/cputype.h>
31#include <asm/debug-monitors.h>
31#include <asm/system_misc.h> 32#include <asm/system_misc.h>
32 33
33/* Determine debug architecture. */ 34/* Determine debug architecture. */
34u8 debug_monitors_arch(void) 35u8 debug_monitors_arch(void)
35{ 36{
36 return read_cpuid(ID_AA64DFR0_EL1) & 0xf; 37 return cpuid_feature_extract_field(read_system_reg(SYS_ID_AA64DFR0_EL1),
38 ID_AA64DFR0_DEBUGVER_SHIFT);
37} 39}
38 40
39/* 41/*
@@ -58,7 +60,7 @@ static u32 mdscr_read(void)
58 * Allow root to disable self-hosted debug from userspace. 60 * Allow root to disable self-hosted debug from userspace.
59 * This is useful if you want to connect an external JTAG debugger. 61 * This is useful if you want to connect an external JTAG debugger.
60 */ 62 */
61static u32 debug_enabled = 1; 63static bool debug_enabled = true;
62 64
63static int create_debug_debugfs_entry(void) 65static int create_debug_debugfs_entry(void)
64{ 66{
@@ -69,7 +71,7 @@ fs_initcall(create_debug_debugfs_entry);
69 71
70static int __init early_debug_disable(char *buf) 72static int __init early_debug_disable(char *buf)
71{ 73{
72 debug_enabled = 0; 74 debug_enabled = false;
73 return 0; 75 return 0;
74} 76}
75 77
diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S
index 8ce9b0577442..a773db92908b 100644
--- a/arch/arm64/kernel/efi-entry.S
+++ b/arch/arm64/kernel/efi-entry.S
@@ -29,7 +29,7 @@
29 * we want to be. The kernel image wants to be placed at TEXT_OFFSET 29 * we want to be. The kernel image wants to be placed at TEXT_OFFSET
30 * from start of RAM. 30 * from start of RAM.
31 */ 31 */
32ENTRY(efi_stub_entry) 32ENTRY(entry)
33 /* 33 /*
34 * Create a stack frame to save FP/LR with extra space 34 * Create a stack frame to save FP/LR with extra space
35 * for image_addr variable passed to efi_entry(). 35 * for image_addr variable passed to efi_entry().
@@ -86,8 +86,8 @@ ENTRY(efi_stub_entry)
86 * entries for the VA range of the current image, so no maintenance is 86 * entries for the VA range of the current image, so no maintenance is
87 * necessary. 87 * necessary.
88 */ 88 */
89 adr x0, efi_stub_entry 89 adr x0, entry
90 adr x1, efi_stub_entry_end 90 adr x1, entry_end
91 sub x1, x1, x0 91 sub x1, x1, x0
92 bl __flush_dcache_area 92 bl __flush_dcache_area
93 93
@@ -120,5 +120,5 @@ efi_load_fail:
120 ldp x29, x30, [sp], #32 120 ldp x29, x30, [sp], #32
121 ret 121 ret
122 122
123efi_stub_entry_end: 123entry_end:
124ENDPROC(efi_stub_entry) 124ENDPROC(entry)
diff --git a/arch/arm64/kernel/efi-stub.c b/arch/arm64/kernel/efi-stub.c
deleted file mode 100644
index 816120ece6bc..000000000000
--- a/arch/arm64/kernel/efi-stub.c
+++ /dev/null
@@ -1,68 +0,0 @@
1/*
2 * Copyright (C) 2013, 2014 Linaro Ltd; <roy.franz@linaro.org>
3 *
4 * This file implements the EFI boot stub for the arm64 kernel.
5 * Adapted from ARM version by Mark Salter <msalter@redhat.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 */
12#include <linux/efi.h>
13#include <asm/efi.h>
14#include <asm/sections.h>
15
16efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table_arg,
17 unsigned long *image_addr,
18 unsigned long *image_size,
19 unsigned long *reserve_addr,
20 unsigned long *reserve_size,
21 unsigned long dram_base,
22 efi_loaded_image_t *image)
23{
24 efi_status_t status;
25 unsigned long kernel_size, kernel_memsize = 0;
26 unsigned long nr_pages;
27 void *old_image_addr = (void *)*image_addr;
28
29 /* Relocate the image, if required. */
30 kernel_size = _edata - _text;
31 if (*image_addr != (dram_base + TEXT_OFFSET)) {
32 kernel_memsize = kernel_size + (_end - _edata);
33
34 /*
35 * First, try a straight allocation at the preferred offset.
36 * This will work around the issue where, if dram_base == 0x0,
37 * efi_low_alloc() refuses to allocate at 0x0 (to prevent the
38 * address of the allocation to be mistaken for a FAIL return
39 * value or a NULL pointer). It will also ensure that, on
40 * platforms where the [dram_base, dram_base + TEXT_OFFSET)
41 * interval is partially occupied by the firmware (like on APM
42 * Mustang), we can still place the kernel at the address
43 * 'dram_base + TEXT_OFFSET'.
44 */
45 *image_addr = *reserve_addr = dram_base + TEXT_OFFSET;
46 nr_pages = round_up(kernel_memsize, EFI_ALLOC_ALIGN) /
47 EFI_PAGE_SIZE;
48 status = efi_call_early(allocate_pages, EFI_ALLOCATE_ADDRESS,
49 EFI_LOADER_DATA, nr_pages,
50 (efi_physical_addr_t *)reserve_addr);
51 if (status != EFI_SUCCESS) {
52 kernel_memsize += TEXT_OFFSET;
53 status = efi_low_alloc(sys_table_arg, kernel_memsize,
54 SZ_2M, reserve_addr);
55
56 if (status != EFI_SUCCESS) {
57 pr_efi_err(sys_table_arg, "Failed to relocate kernel\n");
58 return status;
59 }
60 *image_addr = *reserve_addr + TEXT_OFFSET;
61 }
62 memcpy((void *)*image_addr, old_image_addr, kernel_size);
63 *reserve_size = kernel_memsize;
64 }
65
66
67 return EFI_SUCCESS;
68}
diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c
index 13671a9cf016..fc5508e0df57 100644
--- a/arch/arm64/kernel/efi.c
+++ b/arch/arm64/kernel/efi.c
@@ -48,18 +48,8 @@ static struct mm_struct efi_mm = {
48 .mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem), 48 .mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem),
49 .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock), 49 .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock),
50 .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), 50 .mmlist = LIST_HEAD_INIT(efi_mm.mmlist),
51 INIT_MM_CONTEXT(efi_mm)
52}; 51};
53 52
54static int uefi_debug __initdata;
55static int __init uefi_debug_setup(char *str)
56{
57 uefi_debug = 1;
58
59 return 0;
60}
61early_param("uefi_debug", uefi_debug_setup);
62
63static int __init is_normal_ram(efi_memory_desc_t *md) 53static int __init is_normal_ram(efi_memory_desc_t *md)
64{ 54{
65 if (md->attribute & EFI_MEMORY_WB) 55 if (md->attribute & EFI_MEMORY_WB)
@@ -171,14 +161,14 @@ static __init void reserve_regions(void)
171 efi_memory_desc_t *md; 161 efi_memory_desc_t *md;
172 u64 paddr, npages, size; 162 u64 paddr, npages, size;
173 163
174 if (uefi_debug) 164 if (efi_enabled(EFI_DBG))
175 pr_info("Processing EFI memory map:\n"); 165 pr_info("Processing EFI memory map:\n");
176 166
177 for_each_efi_memory_desc(&memmap, md) { 167 for_each_efi_memory_desc(&memmap, md) {
178 paddr = md->phys_addr; 168 paddr = md->phys_addr;
179 npages = md->num_pages; 169 npages = md->num_pages;
180 170
181 if (uefi_debug) { 171 if (efi_enabled(EFI_DBG)) {
182 char buf[64]; 172 char buf[64];
183 173
184 pr_info(" 0x%012llx-0x%012llx %s", 174 pr_info(" 0x%012llx-0x%012llx %s",
@@ -194,11 +184,11 @@ static __init void reserve_regions(void)
194 184
195 if (is_reserve_region(md)) { 185 if (is_reserve_region(md)) {
196 memblock_reserve(paddr, size); 186 memblock_reserve(paddr, size);
197 if (uefi_debug) 187 if (efi_enabled(EFI_DBG))
198 pr_cont("*"); 188 pr_cont("*");
199 } 189 }
200 190
201 if (uefi_debug) 191 if (efi_enabled(EFI_DBG))
202 pr_cont("\n"); 192 pr_cont("\n");
203 } 193 }
204 194
@@ -210,14 +200,14 @@ void __init efi_init(void)
210 struct efi_fdt_params params; 200 struct efi_fdt_params params;
211 201
212 /* Grab UEFI information placed in FDT by stub */ 202 /* Grab UEFI information placed in FDT by stub */
213 if (!efi_get_fdt_params(&params, uefi_debug)) 203 if (!efi_get_fdt_params(&params))
214 return; 204 return;
215 205
216 efi_system_table = params.system_table; 206 efi_system_table = params.system_table;
217 207
218 memblock_reserve(params.mmap & PAGE_MASK, 208 memblock_reserve(params.mmap & PAGE_MASK,
219 PAGE_ALIGN(params.mmap_size + (params.mmap & ~PAGE_MASK))); 209 PAGE_ALIGN(params.mmap_size + (params.mmap & ~PAGE_MASK)));
220 memmap.phys_map = (void *)params.mmap; 210 memmap.phys_map = params.mmap;
221 memmap.map = early_memremap(params.mmap, params.mmap_size); 211 memmap.map = early_memremap(params.mmap, params.mmap_size);
222 memmap.map_end = memmap.map + params.mmap_size; 212 memmap.map_end = memmap.map + params.mmap_size;
223 memmap.desc_size = params.desc_size; 213 memmap.desc_size = params.desc_size;
@@ -234,6 +224,8 @@ static bool __init efi_virtmap_init(void)
234{ 224{
235 efi_memory_desc_t *md; 225 efi_memory_desc_t *md;
236 226
227 init_new_context(NULL, &efi_mm);
228
237 for_each_efi_memory_desc(&memmap, md) { 229 for_each_efi_memory_desc(&memmap, md) {
238 u64 paddr, npages, size; 230 u64 paddr, npages, size;
239 pgprot_t prot; 231 pgprot_t prot;
@@ -264,7 +256,8 @@ static bool __init efi_virtmap_init(void)
264 else 256 else
265 prot = PAGE_KERNEL; 257 prot = PAGE_KERNEL;
266 258
267 create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot); 259 create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size,
260 __pgprot(pgprot_val(prot) | PTE_NG));
268 } 261 }
269 return true; 262 return true;
270} 263}
@@ -291,7 +284,7 @@ static int __init arm64_enable_runtime_services(void)
291 pr_info("Remapping and enabling EFI services.\n"); 284 pr_info("Remapping and enabling EFI services.\n");
292 285
293 mapsize = memmap.map_end - memmap.map; 286 mapsize = memmap.map_end - memmap.map;
294 memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map, 287 memmap.map = (__force void *)ioremap_cache(memmap.phys_map,
295 mapsize); 288 mapsize);
296 if (!memmap.map) { 289 if (!memmap.map) {
297 pr_err("Failed to remap EFI memory map\n"); 290 pr_err("Failed to remap EFI memory map\n");
@@ -339,14 +332,7 @@ core_initcall(arm64_dmi_init);
339 332
340static void efi_set_pgd(struct mm_struct *mm) 333static void efi_set_pgd(struct mm_struct *mm)
341{ 334{
342 if (mm == &init_mm) 335 switch_mm(NULL, mm, NULL);
343 cpu_set_reserved_ttbr0();
344 else
345 cpu_switch_mm(mm->pgd, mm);
346
347 flush_tlb_all();
348 if (icache_is_aivivt())
349 __flush_icache_all();
350} 336}
351 337
352void efi_virtmap_load(void) 338void efi_virtmap_load(void)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 4306c937b1ff..7ed3d75f6304 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -430,6 +430,8 @@ el0_sync_compat:
430 b.eq el0_fpsimd_acc 430 b.eq el0_fpsimd_acc
431 cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception 431 cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
432 b.eq el0_fpsimd_exc 432 b.eq el0_fpsimd_exc
433 cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
434 b.eq el0_sp_pc
433 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 435 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
434 b.eq el0_undef 436 b.eq el0_undef
435 cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap 437 cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index c56956a16d3f..4c46c54a3ad7 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -332,21 +332,15 @@ static inline void fpsimd_hotplug_init(void) { }
332 */ 332 */
333static int __init fpsimd_init(void) 333static int __init fpsimd_init(void)
334{ 334{
335 u64 pfr = read_cpuid(ID_AA64PFR0_EL1); 335 if (elf_hwcap & HWCAP_FP) {
336 336 fpsimd_pm_init();
337 if (pfr & (0xf << 16)) { 337 fpsimd_hotplug_init();
338 } else {
338 pr_notice("Floating-point is not implemented\n"); 339 pr_notice("Floating-point is not implemented\n");
339 return 0;
340 } 340 }
341 elf_hwcap |= HWCAP_FP;
342 341
343 if (pfr & (0xf << 20)) 342 if (!(elf_hwcap & HWCAP_ASIMD))
344 pr_notice("Advanced SIMD is not implemented\n"); 343 pr_notice("Advanced SIMD is not implemented\n");
345 else
346 elf_hwcap |= HWCAP_ASIMD;
347
348 fpsimd_pm_init();
349 fpsimd_hotplug_init();
350 344
351 return 0; 345 return 0;
352} 346}
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 90d09eddd5b2..23cfc08fc8ba 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -29,11 +29,13 @@
29#include <asm/asm-offsets.h> 29#include <asm/asm-offsets.h>
30#include <asm/cache.h> 30#include <asm/cache.h>
31#include <asm/cputype.h> 31#include <asm/cputype.h>
32#include <asm/kernel-pgtable.h>
32#include <asm/memory.h> 33#include <asm/memory.h>
33#include <asm/thread_info.h>
34#include <asm/pgtable-hwdef.h> 34#include <asm/pgtable-hwdef.h>
35#include <asm/pgtable.h> 35#include <asm/pgtable.h>
36#include <asm/page.h> 36#include <asm/page.h>
37#include <asm/sysreg.h>
38#include <asm/thread_info.h>
37#include <asm/virt.h> 39#include <asm/virt.h>
38 40
39#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET) 41#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET)
@@ -46,32 +48,10 @@
46#error TEXT_OFFSET must be less than 2MB 48#error TEXT_OFFSET must be less than 2MB
47#endif 49#endif
48 50
49#ifdef CONFIG_ARM64_64K_PAGES
50#define BLOCK_SHIFT PAGE_SHIFT
51#define BLOCK_SIZE PAGE_SIZE
52#define TABLE_SHIFT PMD_SHIFT
53#else
54#define BLOCK_SHIFT SECTION_SHIFT
55#define BLOCK_SIZE SECTION_SIZE
56#define TABLE_SHIFT PUD_SHIFT
57#endif
58
59#define KERNEL_START _text 51#define KERNEL_START _text
60#define KERNEL_END _end 52#define KERNEL_END _end
61 53
62/* 54/*
63 * Initial memory map attributes.
64 */
65#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED
66#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S
67
68#ifdef CONFIG_ARM64_64K_PAGES
69#define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS
70#else
71#define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS
72#endif
73
74/*
75 * Kernel startup entry point. 55 * Kernel startup entry point.
76 * --------------------------- 56 * ---------------------------
77 * 57 *
@@ -120,8 +100,8 @@ efi_head:
120#endif 100#endif
121 101
122#ifdef CONFIG_EFI 102#ifdef CONFIG_EFI
123 .globl stext_offset 103 .globl __efistub_stext_offset
124 .set stext_offset, stext - efi_head 104 .set __efistub_stext_offset, stext - efi_head
125 .align 3 105 .align 3
126pe_header: 106pe_header:
127 .ascii "PE" 107 .ascii "PE"
@@ -144,8 +124,8 @@ optional_header:
144 .long _end - stext // SizeOfCode 124 .long _end - stext // SizeOfCode
145 .long 0 // SizeOfInitializedData 125 .long 0 // SizeOfInitializedData
146 .long 0 // SizeOfUninitializedData 126 .long 0 // SizeOfUninitializedData
147 .long efi_stub_entry - efi_head // AddressOfEntryPoint 127 .long __efistub_entry - efi_head // AddressOfEntryPoint
148 .long stext_offset // BaseOfCode 128 .long __efistub_stext_offset // BaseOfCode
149 129
150extra_header_fields: 130extra_header_fields:
151 .quad 0 // ImageBase 131 .quad 0 // ImageBase
@@ -162,7 +142,7 @@ extra_header_fields:
162 .long _end - efi_head // SizeOfImage 142 .long _end - efi_head // SizeOfImage
163 143
164 // Everything before the kernel image is considered part of the header 144 // Everything before the kernel image is considered part of the header
165 .long stext_offset // SizeOfHeaders 145 .long __efistub_stext_offset // SizeOfHeaders
166 .long 0 // CheckSum 146 .long 0 // CheckSum
167 .short 0xa // Subsystem (EFI application) 147 .short 0xa // Subsystem (EFI application)
168 .short 0 // DllCharacteristics 148 .short 0 // DllCharacteristics
@@ -207,9 +187,9 @@ section_table:
207 .byte 0 187 .byte 0
208 .byte 0 // end of 0 padding of section name 188 .byte 0 // end of 0 padding of section name
209 .long _end - stext // VirtualSize 189 .long _end - stext // VirtualSize
210 .long stext_offset // VirtualAddress 190 .long __efistub_stext_offset // VirtualAddress
211 .long _edata - stext // SizeOfRawData 191 .long _edata - stext // SizeOfRawData
212 .long stext_offset // PointerToRawData 192 .long __efistub_stext_offset // PointerToRawData
213 193
214 .long 0 // PointerToRelocations (0 for executables) 194 .long 0 // PointerToRelocations (0 for executables)
215 .long 0 // PointerToLineNumbers (0 for executables) 195 .long 0 // PointerToLineNumbers (0 for executables)
@@ -292,8 +272,11 @@ ENDPROC(preserve_boot_args)
292 */ 272 */
293 .macro create_pgd_entry, tbl, virt, tmp1, tmp2 273 .macro create_pgd_entry, tbl, virt, tmp1, tmp2
294 create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2 274 create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
295#if SWAPPER_PGTABLE_LEVELS == 3 275#if SWAPPER_PGTABLE_LEVELS > 3
296 create_table_entry \tbl, \virt, TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2 276 create_table_entry \tbl, \virt, PUD_SHIFT, PTRS_PER_PUD, \tmp1, \tmp2
277#endif
278#if SWAPPER_PGTABLE_LEVELS > 2
279 create_table_entry \tbl, \virt, SWAPPER_TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
297#endif 280#endif
298 .endm 281 .endm
299 282
@@ -305,15 +288,15 @@ ENDPROC(preserve_boot_args)
305 * Corrupts: phys, start, end, pstate 288 * Corrupts: phys, start, end, pstate
306 */ 289 */
307 .macro create_block_map, tbl, flags, phys, start, end 290 .macro create_block_map, tbl, flags, phys, start, end
308 lsr \phys, \phys, #BLOCK_SHIFT 291 lsr \phys, \phys, #SWAPPER_BLOCK_SHIFT
309 lsr \start, \start, #BLOCK_SHIFT 292 lsr \start, \start, #SWAPPER_BLOCK_SHIFT
310 and \start, \start, #PTRS_PER_PTE - 1 // table index 293 and \start, \start, #PTRS_PER_PTE - 1 // table index
311 orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry 294 orr \phys, \flags, \phys, lsl #SWAPPER_BLOCK_SHIFT // table entry
312 lsr \end, \end, #BLOCK_SHIFT 295 lsr \end, \end, #SWAPPER_BLOCK_SHIFT
313 and \end, \end, #PTRS_PER_PTE - 1 // table end index 296 and \end, \end, #PTRS_PER_PTE - 1 // table end index
3149999: str \phys, [\tbl, \start, lsl #3] // store the entry 2979999: str \phys, [\tbl, \start, lsl #3] // store the entry
315 add \start, \start, #1 // next entry 298 add \start, \start, #1 // next entry
316 add \phys, \phys, #BLOCK_SIZE // next block 299 add \phys, \phys, #SWAPPER_BLOCK_SIZE // next block
317 cmp \start, \end 300 cmp \start, \end
318 b.ls 9999b 301 b.ls 9999b
319 .endm 302 .endm
@@ -350,7 +333,7 @@ __create_page_tables:
350 cmp x0, x6 333 cmp x0, x6
351 b.lo 1b 334 b.lo 1b
352 335
353 ldr x7, =MM_MMUFLAGS 336 ldr x7, =SWAPPER_MM_MMUFLAGS
354 337
355 /* 338 /*
356 * Create the identity mapping. 339 * Create the identity mapping.
@@ -444,6 +427,9 @@ __mmap_switched:
444 str_l x21, __fdt_pointer, x5 // Save FDT pointer 427 str_l x21, __fdt_pointer, x5 // Save FDT pointer
445 str_l x24, memstart_addr, x6 // Save PHYS_OFFSET 428 str_l x24, memstart_addr, x6 // Save PHYS_OFFSET
446 mov x29, #0 429 mov x29, #0
430#ifdef CONFIG_KASAN
431 bl kasan_early_init
432#endif
447 b start_kernel 433 b start_kernel
448ENDPROC(__mmap_switched) 434ENDPROC(__mmap_switched)
449 435
@@ -498,6 +484,8 @@ CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1
498 orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1 484 orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1
499 msr_s ICC_SRE_EL2, x0 485 msr_s ICC_SRE_EL2, x0
500 isb // Make sure SRE is now set 486 isb // Make sure SRE is now set
487 mrs_s x0, ICC_SRE_EL2 // Read SRE back,
488 tbz x0, #0, 3f // and check that it sticks
501 msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults 489 msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults
502 490
5033: 4913:
@@ -628,10 +616,17 @@ ENDPROC(__secondary_switched)
628 * x0 = SCTLR_EL1 value for turning on the MMU. 616 * x0 = SCTLR_EL1 value for turning on the MMU.
629 * x27 = *virtual* address to jump to upon completion 617 * x27 = *virtual* address to jump to upon completion
630 * 618 *
631 * other registers depend on the function called upon completion 619 * Other registers depend on the function called upon completion.
620 *
621 * Checks if the selected granule size is supported by the CPU.
622 * If it isn't, park the CPU
632 */ 623 */
633 .section ".idmap.text", "ax" 624 .section ".idmap.text", "ax"
634__enable_mmu: 625__enable_mmu:
626 mrs x1, ID_AA64MMFR0_EL1
627 ubfx x2, x1, #ID_AA64MMFR0_TGRAN_SHIFT, 4
628 cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED
629 b.ne __no_granule_support
635 ldr x5, =vectors 630 ldr x5, =vectors
636 msr vbar_el1, x5 631 msr vbar_el1, x5
637 msr ttbr0_el1, x25 // load TTBR0 632 msr ttbr0_el1, x25 // load TTBR0
@@ -649,3 +644,8 @@ __enable_mmu:
649 isb 644 isb
650 br x27 645 br x27
651ENDPROC(__enable_mmu) 646ENDPROC(__enable_mmu)
647
648__no_granule_support:
649 wfe
650 b __no_granule_support
651ENDPROC(__no_granule_support)
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index bba85c8f8037..b45c95d34b83 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -28,6 +28,7 @@
28#include <linux/ptrace.h> 28#include <linux/ptrace.h>
29#include <linux/smp.h> 29#include <linux/smp.h>
30 30
31#include <asm/compat.h>
31#include <asm/current.h> 32#include <asm/current.h>
32#include <asm/debug-monitors.h> 33#include <asm/debug-monitors.h>
33#include <asm/hw_breakpoint.h> 34#include <asm/hw_breakpoint.h>
@@ -163,6 +164,20 @@ enum hw_breakpoint_ops {
163 HW_BREAKPOINT_RESTORE 164 HW_BREAKPOINT_RESTORE
164}; 165};
165 166
167static int is_compat_bp(struct perf_event *bp)
168{
169 struct task_struct *tsk = bp->hw.target;
170
171 /*
172 * tsk can be NULL for per-cpu (non-ptrace) breakpoints.
173 * In this case, use the native interface, since we don't have
174 * the notion of a "compat CPU" and could end up relying on
175 * deprecated behaviour if we use unaligned watchpoints in
176 * AArch64 state.
177 */
178 return tsk && is_compat_thread(task_thread_info(tsk));
179}
180
166/** 181/**
167 * hw_breakpoint_slot_setup - Find and setup a perf slot according to 182 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
168 * operations 183 * operations
@@ -420,7 +435,7 @@ static int arch_build_bp_info(struct perf_event *bp)
420 * Watchpoints can be of length 1, 2, 4 or 8 bytes. 435 * Watchpoints can be of length 1, 2, 4 or 8 bytes.
421 */ 436 */
422 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { 437 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
423 if (is_compat_task()) { 438 if (is_compat_bp(bp)) {
424 if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 && 439 if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
425 info->ctrl.len != ARM_BREAKPOINT_LEN_4) 440 info->ctrl.len != ARM_BREAKPOINT_LEN_4)
426 return -EINVAL; 441 return -EINVAL;
@@ -477,7 +492,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp)
477 * AArch32 tasks expect some simple alignment fixups, so emulate 492 * AArch32 tasks expect some simple alignment fixups, so emulate
478 * that here. 493 * that here.
479 */ 494 */
480 if (is_compat_task()) { 495 if (is_compat_bp(bp)) {
481 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) 496 if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
482 alignment_mask = 0x7; 497 alignment_mask = 0x7;
483 else 498 else
diff --git a/arch/arm64/kernel/image.h b/arch/arm64/kernel/image.h
index 8fae0756e175..bc2abb8b1599 100644
--- a/arch/arm64/kernel/image.h
+++ b/arch/arm64/kernel/image.h
@@ -47,7 +47,10 @@
47#define __HEAD_FLAG_BE 0 47#define __HEAD_FLAG_BE 0
48#endif 48#endif
49 49
50#define __HEAD_FLAGS (__HEAD_FLAG_BE << 0) 50#define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2)
51
52#define __HEAD_FLAGS ((__HEAD_FLAG_BE << 0) | \
53 (__HEAD_FLAG_PAGE_SIZE << 1))
51 54
52/* 55/*
53 * These will output as part of the Image header, which should be little-endian 56 * These will output as part of the Image header, which should be little-endian
@@ -59,4 +62,37 @@
59 _kernel_offset_le = DATA_LE64(TEXT_OFFSET); \ 62 _kernel_offset_le = DATA_LE64(TEXT_OFFSET); \
60 _kernel_flags_le = DATA_LE64(__HEAD_FLAGS); 63 _kernel_flags_le = DATA_LE64(__HEAD_FLAGS);
61 64
65#ifdef CONFIG_EFI
66
67/*
68 * The EFI stub has its own symbol namespace prefixed by __efistub_, to
69 * isolate it from the kernel proper. The following symbols are legally
70 * accessed by the stub, so provide some aliases to make them accessible.
71 * Only include data symbols here, or text symbols of functions that are
72 * guaranteed to be safe when executed at another offset than they were
73 * linked at. The routines below are all implemented in assembler in a
74 * position independent manner
75 */
76__efistub_memcmp = __pi_memcmp;
77__efistub_memchr = __pi_memchr;
78__efistub_memcpy = __pi_memcpy;
79__efistub_memmove = __pi_memmove;
80__efistub_memset = __pi_memset;
81__efistub_strlen = __pi_strlen;
82__efistub_strcmp = __pi_strcmp;
83__efistub_strncmp = __pi_strncmp;
84__efistub___flush_dcache_area = __pi___flush_dcache_area;
85
86#ifdef CONFIG_KASAN
87__efistub___memcpy = __pi_memcpy;
88__efistub___memmove = __pi_memmove;
89__efistub___memset = __pi_memset;
90#endif
91
92__efistub__text = _text;
93__efistub__end = _end;
94__efistub__edata = _edata;
95
96#endif
97
62#endif /* __ASM_IMAGE_H */ 98#endif /* __ASM_IMAGE_H */
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 11dc3fd47853..9f17ec071ee0 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -27,7 +27,6 @@
27#include <linux/init.h> 27#include <linux/init.h>
28#include <linux/irqchip.h> 28#include <linux/irqchip.h>
29#include <linux/seq_file.h> 29#include <linux/seq_file.h>
30#include <linux/ratelimit.h>
31 30
32unsigned long irq_err_count; 31unsigned long irq_err_count;
33 32
@@ -54,64 +53,3 @@ void __init init_IRQ(void)
54 if (!handle_arch_irq) 53 if (!handle_arch_irq)
55 panic("No interrupt controller found."); 54 panic("No interrupt controller found.");
56} 55}
57
58#ifdef CONFIG_HOTPLUG_CPU
59static bool migrate_one_irq(struct irq_desc *desc)
60{
61 struct irq_data *d = irq_desc_get_irq_data(desc);
62 const struct cpumask *affinity = irq_data_get_affinity_mask(d);
63 struct irq_chip *c;
64 bool ret = false;
65
66 /*
67 * If this is a per-CPU interrupt, or the affinity does not
68 * include this CPU, then we have nothing to do.
69 */
70 if (irqd_is_per_cpu(d) || !cpumask_test_cpu(smp_processor_id(), affinity))
71 return false;
72
73 if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
74 affinity = cpu_online_mask;
75 ret = true;
76 }
77
78 c = irq_data_get_irq_chip(d);
79 if (!c->irq_set_affinity)
80 pr_debug("IRQ%u: unable to set affinity\n", d->irq);
81 else if (c->irq_set_affinity(d, affinity, false) == IRQ_SET_MASK_OK && ret)
82 cpumask_copy(irq_data_get_affinity_mask(d), affinity);
83
84 return ret;
85}
86
87/*
88 * The current CPU has been marked offline. Migrate IRQs off this CPU.
89 * If the affinity settings do not allow other CPUs, force them onto any
90 * available CPU.
91 *
92 * Note: we must iterate over all IRQs, whether they have an attached
93 * action structure or not, as we need to get chained interrupts too.
94 */
95void migrate_irqs(void)
96{
97 unsigned int i;
98 struct irq_desc *desc;
99 unsigned long flags;
100
101 local_irq_save(flags);
102
103 for_each_irq_desc(i, desc) {
104 bool affinity_broken;
105
106 raw_spin_lock(&desc->lock);
107 affinity_broken = migrate_one_irq(desc);
108 raw_spin_unlock(&desc->lock);
109
110 if (affinity_broken)
111 pr_warn_ratelimited("IRQ%u no longer affine to CPU%u\n",
112 i, smp_processor_id());
113 }
114
115 local_irq_restore(flags);
116}
117#endif /* CONFIG_HOTPLUG_CPU */
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index 876eb8df50bf..f4bc779e62e8 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -21,6 +21,7 @@
21#include <linux/bitops.h> 21#include <linux/bitops.h>
22#include <linux/elf.h> 22#include <linux/elf.h>
23#include <linux/gfp.h> 23#include <linux/gfp.h>
24#include <linux/kasan.h>
24#include <linux/kernel.h> 25#include <linux/kernel.h>
25#include <linux/mm.h> 26#include <linux/mm.h>
26#include <linux/moduleloader.h> 27#include <linux/moduleloader.h>
@@ -34,9 +35,18 @@
34 35
35void *module_alloc(unsigned long size) 36void *module_alloc(unsigned long size)
36{ 37{
37 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, 38 void *p;
38 GFP_KERNEL, PAGE_KERNEL_EXEC, 0, 39
39 NUMA_NO_NODE, __builtin_return_address(0)); 40 p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END,
41 GFP_KERNEL, PAGE_KERNEL_EXEC, 0,
42 NUMA_NO_NODE, __builtin_return_address(0));
43
44 if (p && (kasan_module_alloc(p, size) < 0)) {
45 vfree(p);
46 return NULL;
47 }
48
49 return p;
40} 50}
41 51
42enum aarch64_reloc_op { 52enum aarch64_reloc_op {
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index f9a74d4fff3b..5b1897e8ca24 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -18,651 +18,12 @@
18 * You should have received a copy of the GNU General Public License 18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>. 19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 */ 20 */
21#define pr_fmt(fmt) "hw perfevents: " fmt
22
23#include <linux/bitmap.h>
24#include <linux/interrupt.h>
25#include <linux/irq.h>
26#include <linux/kernel.h>
27#include <linux/export.h>
28#include <linux/of_device.h>
29#include <linux/perf_event.h>
30#include <linux/platform_device.h>
31#include <linux/slab.h>
32#include <linux/spinlock.h>
33#include <linux/uaccess.h>
34 21
35#include <asm/cputype.h>
36#include <asm/irq.h>
37#include <asm/irq_regs.h> 22#include <asm/irq_regs.h>
38#include <asm/pmu.h>
39
40/*
41 * ARMv8 supports a maximum of 32 events.
42 * The cycle counter is included in this total.
43 */
44#define ARMPMU_MAX_HWEVENTS 32
45
46static DEFINE_PER_CPU(struct perf_event * [ARMPMU_MAX_HWEVENTS], hw_events);
47static DEFINE_PER_CPU(unsigned long [BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)], used_mask);
48static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events);
49
50#define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
51
52/* Set at runtime when we know what CPU type we are. */
53static struct arm_pmu *cpu_pmu;
54
55int
56armpmu_get_max_events(void)
57{
58 int max_events = 0;
59
60 if (cpu_pmu != NULL)
61 max_events = cpu_pmu->num_events;
62
63 return max_events;
64}
65EXPORT_SYMBOL_GPL(armpmu_get_max_events);
66
67int perf_num_counters(void)
68{
69 return armpmu_get_max_events();
70}
71EXPORT_SYMBOL_GPL(perf_num_counters);
72
73#define HW_OP_UNSUPPORTED 0xFFFF
74
75#define C(_x) \
76 PERF_COUNT_HW_CACHE_##_x
77
78#define CACHE_OP_UNSUPPORTED 0xFFFF
79
80#define PERF_MAP_ALL_UNSUPPORTED \
81 [0 ... PERF_COUNT_HW_MAX - 1] = HW_OP_UNSUPPORTED
82
83#define PERF_CACHE_MAP_ALL_UNSUPPORTED \
84[0 ... C(MAX) - 1] = { \
85 [0 ... C(OP_MAX) - 1] = { \
86 [0 ... C(RESULT_MAX) - 1] = CACHE_OP_UNSUPPORTED, \
87 }, \
88}
89
90static int
91armpmu_map_cache_event(const unsigned (*cache_map)
92 [PERF_COUNT_HW_CACHE_MAX]
93 [PERF_COUNT_HW_CACHE_OP_MAX]
94 [PERF_COUNT_HW_CACHE_RESULT_MAX],
95 u64 config)
96{
97 unsigned int cache_type, cache_op, cache_result, ret;
98
99 cache_type = (config >> 0) & 0xff;
100 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
101 return -EINVAL;
102
103 cache_op = (config >> 8) & 0xff;
104 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
105 return -EINVAL;
106
107 cache_result = (config >> 16) & 0xff;
108 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
109 return -EINVAL;
110
111 ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
112
113 if (ret == CACHE_OP_UNSUPPORTED)
114 return -ENOENT;
115
116 return ret;
117}
118
119static int
120armpmu_map_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
121{
122 int mapping;
123
124 if (config >= PERF_COUNT_HW_MAX)
125 return -EINVAL;
126
127 mapping = (*event_map)[config];
128 return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
129}
130
131static int
132armpmu_map_raw_event(u32 raw_event_mask, u64 config)
133{
134 return (int)(config & raw_event_mask);
135}
136
137static int map_cpu_event(struct perf_event *event,
138 const unsigned (*event_map)[PERF_COUNT_HW_MAX],
139 const unsigned (*cache_map)
140 [PERF_COUNT_HW_CACHE_MAX]
141 [PERF_COUNT_HW_CACHE_OP_MAX]
142 [PERF_COUNT_HW_CACHE_RESULT_MAX],
143 u32 raw_event_mask)
144{
145 u64 config = event->attr.config;
146
147 switch (event->attr.type) {
148 case PERF_TYPE_HARDWARE:
149 return armpmu_map_event(event_map, config);
150 case PERF_TYPE_HW_CACHE:
151 return armpmu_map_cache_event(cache_map, config);
152 case PERF_TYPE_RAW:
153 return armpmu_map_raw_event(raw_event_mask, config);
154 }
155
156 return -ENOENT;
157}
158
159int
160armpmu_event_set_period(struct perf_event *event,
161 struct hw_perf_event *hwc,
162 int idx)
163{
164 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
165 s64 left = local64_read(&hwc->period_left);
166 s64 period = hwc->sample_period;
167 int ret = 0;
168
169 if (unlikely(left <= -period)) {
170 left = period;
171 local64_set(&hwc->period_left, left);
172 hwc->last_period = period;
173 ret = 1;
174 }
175
176 if (unlikely(left <= 0)) {
177 left += period;
178 local64_set(&hwc->period_left, left);
179 hwc->last_period = period;
180 ret = 1;
181 }
182
183 /*
184 * Limit the maximum period to prevent the counter value
185 * from overtaking the one we are about to program. In
186 * effect we are reducing max_period to account for
187 * interrupt latency (and we are being very conservative).
188 */
189 if (left > (armpmu->max_period >> 1))
190 left = armpmu->max_period >> 1;
191
192 local64_set(&hwc->prev_count, (u64)-left);
193
194 armpmu->write_counter(idx, (u64)(-left) & 0xffffffff);
195
196 perf_event_update_userpage(event);
197
198 return ret;
199}
200
201u64
202armpmu_event_update(struct perf_event *event,
203 struct hw_perf_event *hwc,
204 int idx)
205{
206 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
207 u64 delta, prev_raw_count, new_raw_count;
208
209again:
210 prev_raw_count = local64_read(&hwc->prev_count);
211 new_raw_count = armpmu->read_counter(idx);
212
213 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
214 new_raw_count) != prev_raw_count)
215 goto again;
216
217 delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
218
219 local64_add(delta, &event->count);
220 local64_sub(delta, &hwc->period_left);
221
222 return new_raw_count;
223}
224
225static void
226armpmu_read(struct perf_event *event)
227{
228 struct hw_perf_event *hwc = &event->hw;
229
230 /* Don't read disabled counters! */
231 if (hwc->idx < 0)
232 return;
233
234 armpmu_event_update(event, hwc, hwc->idx);
235}
236
237static void
238armpmu_stop(struct perf_event *event, int flags)
239{
240 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
241 struct hw_perf_event *hwc = &event->hw;
242
243 /*
244 * ARM pmu always has to update the counter, so ignore
245 * PERF_EF_UPDATE, see comments in armpmu_start().
246 */
247 if (!(hwc->state & PERF_HES_STOPPED)) {
248 armpmu->disable(hwc, hwc->idx);
249 barrier(); /* why? */
250 armpmu_event_update(event, hwc, hwc->idx);
251 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
252 }
253}
254
255static void
256armpmu_start(struct perf_event *event, int flags)
257{
258 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
259 struct hw_perf_event *hwc = &event->hw;
260
261 /*
262 * ARM pmu always has to reprogram the period, so ignore
263 * PERF_EF_RELOAD, see the comment below.
264 */
265 if (flags & PERF_EF_RELOAD)
266 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
267
268 hwc->state = 0;
269 /*
270 * Set the period again. Some counters can't be stopped, so when we
271 * were stopped we simply disabled the IRQ source and the counter
272 * may have been left counting. If we don't do this step then we may
273 * get an interrupt too soon or *way* too late if the overflow has
274 * happened since disabling.
275 */
276 armpmu_event_set_period(event, hwc, hwc->idx);
277 armpmu->enable(hwc, hwc->idx);
278}
279
280static void
281armpmu_del(struct perf_event *event, int flags)
282{
283 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
284 struct pmu_hw_events *hw_events = armpmu->get_hw_events();
285 struct hw_perf_event *hwc = &event->hw;
286 int idx = hwc->idx;
287
288 WARN_ON(idx < 0);
289
290 armpmu_stop(event, PERF_EF_UPDATE);
291 hw_events->events[idx] = NULL;
292 clear_bit(idx, hw_events->used_mask);
293
294 perf_event_update_userpage(event);
295}
296
297static int
298armpmu_add(struct perf_event *event, int flags)
299{
300 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
301 struct pmu_hw_events *hw_events = armpmu->get_hw_events();
302 struct hw_perf_event *hwc = &event->hw;
303 int idx;
304 int err = 0;
305
306 perf_pmu_disable(event->pmu);
307
308 /* If we don't have a space for the counter then finish early. */
309 idx = armpmu->get_event_idx(hw_events, hwc);
310 if (idx < 0) {
311 err = idx;
312 goto out;
313 }
314
315 /*
316 * If there is an event in the counter we are going to use then make
317 * sure it is disabled.
318 */
319 event->hw.idx = idx;
320 armpmu->disable(hwc, idx);
321 hw_events->events[idx] = event;
322
323 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
324 if (flags & PERF_EF_START)
325 armpmu_start(event, PERF_EF_RELOAD);
326
327 /* Propagate our changes to the userspace mapping. */
328 perf_event_update_userpage(event);
329
330out:
331 perf_pmu_enable(event->pmu);
332 return err;
333}
334
335static int
336validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events,
337 struct perf_event *event)
338{
339 struct arm_pmu *armpmu;
340 struct hw_perf_event fake_event = event->hw;
341 struct pmu *leader_pmu = event->group_leader->pmu;
342
343 if (is_software_event(event))
344 return 1;
345
346 /*
347 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
348 * core perf code won't check that the pmu->ctx == leader->ctx
349 * until after pmu->event_init(event).
350 */
351 if (event->pmu != pmu)
352 return 0;
353
354 if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF)
355 return 1;
356
357 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
358 return 1;
359
360 armpmu = to_arm_pmu(event->pmu);
361 return armpmu->get_event_idx(hw_events, &fake_event) >= 0;
362}
363
364static int
365validate_group(struct perf_event *event)
366{
367 struct perf_event *sibling, *leader = event->group_leader;
368 struct pmu_hw_events fake_pmu;
369 DECLARE_BITMAP(fake_used_mask, ARMPMU_MAX_HWEVENTS);
370
371 /*
372 * Initialise the fake PMU. We only need to populate the
373 * used_mask for the purposes of validation.
374 */
375 memset(fake_used_mask, 0, sizeof(fake_used_mask));
376 fake_pmu.used_mask = fake_used_mask;
377
378 if (!validate_event(event->pmu, &fake_pmu, leader))
379 return -EINVAL;
380
381 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
382 if (!validate_event(event->pmu, &fake_pmu, sibling))
383 return -EINVAL;
384 }
385
386 if (!validate_event(event->pmu, &fake_pmu, event))
387 return -EINVAL;
388
389 return 0;
390}
391
392static void
393armpmu_disable_percpu_irq(void *data)
394{
395 unsigned int irq = *(unsigned int *)data;
396 disable_percpu_irq(irq);
397}
398
399static void
400armpmu_release_hardware(struct arm_pmu *armpmu)
401{
402 int irq;
403 unsigned int i, irqs;
404 struct platform_device *pmu_device = armpmu->plat_device;
405
406 irqs = min(pmu_device->num_resources, num_possible_cpus());
407 if (!irqs)
408 return;
409
410 irq = platform_get_irq(pmu_device, 0);
411 if (irq <= 0)
412 return;
413
414 if (irq_is_percpu(irq)) {
415 on_each_cpu(armpmu_disable_percpu_irq, &irq, 1);
416 free_percpu_irq(irq, &cpu_hw_events);
417 } else {
418 for (i = 0; i < irqs; ++i) {
419 int cpu = i;
420
421 if (armpmu->irq_affinity)
422 cpu = armpmu->irq_affinity[i];
423
424 if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs))
425 continue;
426 irq = platform_get_irq(pmu_device, i);
427 if (irq > 0)
428 free_irq(irq, armpmu);
429 }
430 }
431}
432
433static void
434armpmu_enable_percpu_irq(void *data)
435{
436 unsigned int irq = *(unsigned int *)data;
437 enable_percpu_irq(irq, IRQ_TYPE_NONE);
438}
439
440static int
441armpmu_reserve_hardware(struct arm_pmu *armpmu)
442{
443 int err, irq;
444 unsigned int i, irqs;
445 struct platform_device *pmu_device = armpmu->plat_device;
446
447 if (!pmu_device)
448 return -ENODEV;
449
450 irqs = min(pmu_device->num_resources, num_possible_cpus());
451 if (!irqs) {
452 pr_err("no irqs for PMUs defined\n");
453 return -ENODEV;
454 }
455
456 irq = platform_get_irq(pmu_device, 0);
457 if (irq <= 0) {
458 pr_err("failed to get valid irq for PMU device\n");
459 return -ENODEV;
460 }
461
462 if (irq_is_percpu(irq)) {
463 err = request_percpu_irq(irq, armpmu->handle_irq,
464 "arm-pmu", &cpu_hw_events);
465
466 if (err) {
467 pr_err("unable to request percpu IRQ%d for ARM PMU counters\n",
468 irq);
469 armpmu_release_hardware(armpmu);
470 return err;
471 }
472
473 on_each_cpu(armpmu_enable_percpu_irq, &irq, 1);
474 } else {
475 for (i = 0; i < irqs; ++i) {
476 int cpu = i;
477
478 err = 0;
479 irq = platform_get_irq(pmu_device, i);
480 if (irq <= 0)
481 continue;
482
483 if (armpmu->irq_affinity)
484 cpu = armpmu->irq_affinity[i];
485
486 /*
487 * If we have a single PMU interrupt that we can't shift,
488 * assume that we're running on a uniprocessor machine and
489 * continue. Otherwise, continue without this interrupt.
490 */
491 if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {
492 pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
493 irq, cpu);
494 continue;
495 }
496
497 err = request_irq(irq, armpmu->handle_irq,
498 IRQF_NOBALANCING | IRQF_NO_THREAD,
499 "arm-pmu", armpmu);
500 if (err) {
501 pr_err("unable to request IRQ%d for ARM PMU counters\n",
502 irq);
503 armpmu_release_hardware(armpmu);
504 return err;
505 }
506
507 cpumask_set_cpu(cpu, &armpmu->active_irqs);
508 }
509 }
510
511 return 0;
512}
513
514static void
515hw_perf_event_destroy(struct perf_event *event)
516{
517 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
518 atomic_t *active_events = &armpmu->active_events;
519 struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;
520
521 if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
522 armpmu_release_hardware(armpmu);
523 mutex_unlock(pmu_reserve_mutex);
524 }
525}
526
527static int
528event_requires_mode_exclusion(struct perf_event_attr *attr)
529{
530 return attr->exclude_idle || attr->exclude_user ||
531 attr->exclude_kernel || attr->exclude_hv;
532}
533
534static int
535__hw_perf_event_init(struct perf_event *event)
536{
537 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
538 struct hw_perf_event *hwc = &event->hw;
539 int mapping, err;
540
541 mapping = armpmu->map_event(event);
542
543 if (mapping < 0) {
544 pr_debug("event %x:%llx not supported\n", event->attr.type,
545 event->attr.config);
546 return mapping;
547 }
548
549 /*
550 * We don't assign an index until we actually place the event onto
551 * hardware. Use -1 to signify that we haven't decided where to put it
552 * yet. For SMP systems, each core has it's own PMU so we can't do any
553 * clever allocation or constraints checking at this point.
554 */
555 hwc->idx = -1;
556 hwc->config_base = 0;
557 hwc->config = 0;
558 hwc->event_base = 0;
559
560 /*
561 * Check whether we need to exclude the counter from certain modes.
562 */
563 if ((!armpmu->set_event_filter ||
564 armpmu->set_event_filter(hwc, &event->attr)) &&
565 event_requires_mode_exclusion(&event->attr)) {
566 pr_debug("ARM performance counters do not support mode exclusion\n");
567 return -EPERM;
568 }
569
570 /*
571 * Store the event encoding into the config_base field.
572 */
573 hwc->config_base |= (unsigned long)mapping;
574
575 if (!hwc->sample_period) {
576 /*
577 * For non-sampling runs, limit the sample_period to half
578 * of the counter width. That way, the new counter value
579 * is far less likely to overtake the previous one unless
580 * you have some serious IRQ latency issues.
581 */
582 hwc->sample_period = armpmu->max_period >> 1;
583 hwc->last_period = hwc->sample_period;
584 local64_set(&hwc->period_left, hwc->sample_period);
585 }
586
587 err = 0;
588 if (event->group_leader != event) {
589 err = validate_group(event);
590 if (err)
591 return -EINVAL;
592 }
593
594 return err;
595}
596
597static int armpmu_event_init(struct perf_event *event)
598{
599 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
600 int err = 0;
601 atomic_t *active_events = &armpmu->active_events;
602
603 if (armpmu->map_event(event) == -ENOENT)
604 return -ENOENT;
605
606 event->destroy = hw_perf_event_destroy;
607
608 if (!atomic_inc_not_zero(active_events)) {
609 mutex_lock(&armpmu->reserve_mutex);
610 if (atomic_read(active_events) == 0)
611 err = armpmu_reserve_hardware(armpmu);
612
613 if (!err)
614 atomic_inc(active_events);
615 mutex_unlock(&armpmu->reserve_mutex);
616 }
617 23
618 if (err) 24#include <linux/of.h>
619 return err; 25#include <linux/perf/arm_pmu.h>
620 26#include <linux/platform_device.h>
621 err = __hw_perf_event_init(event);
622 if (err)
623 hw_perf_event_destroy(event);
624
625 return err;
626}
627
628static void armpmu_enable(struct pmu *pmu)
629{
630 struct arm_pmu *armpmu = to_arm_pmu(pmu);
631 struct pmu_hw_events *hw_events = armpmu->get_hw_events();
632 int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);
633
634 if (enabled)
635 armpmu->start();
636}
637
638static void armpmu_disable(struct pmu *pmu)
639{
640 struct arm_pmu *armpmu = to_arm_pmu(pmu);
641 armpmu->stop();
642}
643
644static void __init armpmu_init(struct arm_pmu *armpmu)
645{
646 atomic_set(&armpmu->active_events, 0);
647 mutex_init(&armpmu->reserve_mutex);
648
649 armpmu->pmu = (struct pmu) {
650 .pmu_enable = armpmu_enable,
651 .pmu_disable = armpmu_disable,
652 .event_init = armpmu_event_init,
653 .add = armpmu_add,
654 .del = armpmu_del,
655 .start = armpmu_start,
656 .stop = armpmu_stop,
657 .read = armpmu_read,
658 };
659}
660
661int __init armpmu_register(struct arm_pmu *armpmu, char *name, int type)
662{
663 armpmu_init(armpmu);
664 return perf_pmu_register(&armpmu->pmu, name, type);
665}
666 27
667/* 28/*
668 * ARMv8 PMUv3 Performance Events handling code. 29 * ARMv8 PMUv3 Performance Events handling code.
@@ -708,6 +69,21 @@ enum armv8_pmuv3_perf_types {
708 ARMV8_PMUV3_PERFCTR_BUS_CYCLES = 0x1D, 69 ARMV8_PMUV3_PERFCTR_BUS_CYCLES = 0x1D,
709}; 70};
710 71
72/* ARMv8 Cortex-A53 specific event types. */
73enum armv8_a53_pmu_perf_types {
74 ARMV8_A53_PERFCTR_PREFETCH_LINEFILL = 0xC2,
75};
76
77/* ARMv8 Cortex-A57 specific event types. */
78enum armv8_a57_perf_types {
79 ARMV8_A57_PERFCTR_L1_DCACHE_ACCESS_LD = 0x40,
80 ARMV8_A57_PERFCTR_L1_DCACHE_ACCESS_ST = 0x41,
81 ARMV8_A57_PERFCTR_L1_DCACHE_REFILL_LD = 0x42,
82 ARMV8_A57_PERFCTR_L1_DCACHE_REFILL_ST = 0x43,
83 ARMV8_A57_PERFCTR_DTLB_REFILL_LD = 0x4c,
84 ARMV8_A57_PERFCTR_DTLB_REFILL_ST = 0x4d,
85};
86
711/* PMUv3 HW events mapping. */ 87/* PMUv3 HW events mapping. */
712static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = { 88static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
713 PERF_MAP_ALL_UNSUPPORTED, 89 PERF_MAP_ALL_UNSUPPORTED,
@@ -718,6 +94,28 @@ static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
718 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED, 94 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
719}; 95};
720 96
97/* ARM Cortex-A53 HW events mapping. */
98static const unsigned armv8_a53_perf_map[PERF_COUNT_HW_MAX] = {
99 PERF_MAP_ALL_UNSUPPORTED,
100 [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
101 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
102 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
103 [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
104 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE,
105 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
106 [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
107};
108
109static const unsigned armv8_a57_perf_map[PERF_COUNT_HW_MAX] = {
110 PERF_MAP_ALL_UNSUPPORTED,
111 [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
112 [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
113 [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
114 [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
115 [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
116 [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
117};
118
721static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] 119static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
722 [PERF_COUNT_HW_CACHE_OP_MAX] 120 [PERF_COUNT_HW_CACHE_OP_MAX]
723 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 121 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
@@ -734,12 +132,60 @@ static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
734 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED, 132 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
735}; 133};
736 134
135static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
136 [PERF_COUNT_HW_CACHE_OP_MAX]
137 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
138 PERF_CACHE_MAP_ALL_UNSUPPORTED,
139
140 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
141 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
142 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
143 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
144 [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREFETCH_LINEFILL,
145
146 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS,
147 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL,
148
149 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_ITLB_REFILL,
150
151 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
152 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
153 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
154 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
155};
156
157static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
158 [PERF_COUNT_HW_CACHE_OP_MAX]
159 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
160 PERF_CACHE_MAP_ALL_UNSUPPORTED,
161
162 [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_A57_PERFCTR_L1_DCACHE_ACCESS_LD,
163 [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_A57_PERFCTR_L1_DCACHE_REFILL_LD,
164 [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_A57_PERFCTR_L1_DCACHE_ACCESS_ST,
165 [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_A57_PERFCTR_L1_DCACHE_REFILL_ST,
166
167 [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS,
168 [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL,
169
170 [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_A57_PERFCTR_DTLB_REFILL_LD,
171 [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_A57_PERFCTR_DTLB_REFILL_ST,
172
173 [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_ITLB_REFILL,
174
175 [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
176 [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
177 [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
178 [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
179};
180
181
737/* 182/*
738 * Perf Events' indices 183 * Perf Events' indices
739 */ 184 */
740#define ARMV8_IDX_CYCLE_COUNTER 0 185#define ARMV8_IDX_CYCLE_COUNTER 0
741#define ARMV8_IDX_COUNTER0 1 186#define ARMV8_IDX_COUNTER0 1
742#define ARMV8_IDX_COUNTER_LAST (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1) 187#define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
188 (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
743 189
744#define ARMV8_MAX_COUNTERS 32 190#define ARMV8_MAX_COUNTERS 32
745#define ARMV8_COUNTER_MASK (ARMV8_MAX_COUNTERS - 1) 191#define ARMV8_COUNTER_MASK (ARMV8_MAX_COUNTERS - 1)
@@ -805,49 +251,34 @@ static inline int armv8pmu_has_overflowed(u32 pmovsr)
805 return pmovsr & ARMV8_OVERFLOWED_MASK; 251 return pmovsr & ARMV8_OVERFLOWED_MASK;
806} 252}
807 253
808static inline int armv8pmu_counter_valid(int idx) 254static inline int armv8pmu_counter_valid(struct arm_pmu *cpu_pmu, int idx)
809{ 255{
810 return idx >= ARMV8_IDX_CYCLE_COUNTER && idx <= ARMV8_IDX_COUNTER_LAST; 256 return idx >= ARMV8_IDX_CYCLE_COUNTER &&
257 idx <= ARMV8_IDX_COUNTER_LAST(cpu_pmu);
811} 258}
812 259
813static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx) 260static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
814{ 261{
815 int ret = 0; 262 return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
816 u32 counter;
817
818 if (!armv8pmu_counter_valid(idx)) {
819 pr_err("CPU%u checking wrong counter %d overflow status\n",
820 smp_processor_id(), idx);
821 } else {
822 counter = ARMV8_IDX_TO_COUNTER(idx);
823 ret = pmnc & BIT(counter);
824 }
825
826 return ret;
827} 263}
828 264
829static inline int armv8pmu_select_counter(int idx) 265static inline int armv8pmu_select_counter(int idx)
830{ 266{
831 u32 counter; 267 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
832
833 if (!armv8pmu_counter_valid(idx)) {
834 pr_err("CPU%u selecting wrong PMNC counter %d\n",
835 smp_processor_id(), idx);
836 return -EINVAL;
837 }
838
839 counter = ARMV8_IDX_TO_COUNTER(idx);
840 asm volatile("msr pmselr_el0, %0" :: "r" (counter)); 268 asm volatile("msr pmselr_el0, %0" :: "r" (counter));
841 isb(); 269 isb();
842 270
843 return idx; 271 return idx;
844} 272}
845 273
846static inline u32 armv8pmu_read_counter(int idx) 274static inline u32 armv8pmu_read_counter(struct perf_event *event)
847{ 275{
276 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
277 struct hw_perf_event *hwc = &event->hw;
278 int idx = hwc->idx;
848 u32 value = 0; 279 u32 value = 0;
849 280
850 if (!armv8pmu_counter_valid(idx)) 281 if (!armv8pmu_counter_valid(cpu_pmu, idx))
851 pr_err("CPU%u reading wrong counter %d\n", 282 pr_err("CPU%u reading wrong counter %d\n",
852 smp_processor_id(), idx); 283 smp_processor_id(), idx);
853 else if (idx == ARMV8_IDX_CYCLE_COUNTER) 284 else if (idx == ARMV8_IDX_CYCLE_COUNTER)
@@ -858,9 +289,13 @@ static inline u32 armv8pmu_read_counter(int idx)
858 return value; 289 return value;
859} 290}
860 291
861static inline void armv8pmu_write_counter(int idx, u32 value) 292static inline void armv8pmu_write_counter(struct perf_event *event, u32 value)
862{ 293{
863 if (!armv8pmu_counter_valid(idx)) 294 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
295 struct hw_perf_event *hwc = &event->hw;
296 int idx = hwc->idx;
297
298 if (!armv8pmu_counter_valid(cpu_pmu, idx))
864 pr_err("CPU%u writing wrong counter %d\n", 299 pr_err("CPU%u writing wrong counter %d\n",
865 smp_processor_id(), idx); 300 smp_processor_id(), idx);
866 else if (idx == ARMV8_IDX_CYCLE_COUNTER) 301 else if (idx == ARMV8_IDX_CYCLE_COUNTER)
@@ -879,65 +314,34 @@ static inline void armv8pmu_write_evtype(int idx, u32 val)
879 314
880static inline int armv8pmu_enable_counter(int idx) 315static inline int armv8pmu_enable_counter(int idx)
881{ 316{
882 u32 counter; 317 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
883
884 if (!armv8pmu_counter_valid(idx)) {
885 pr_err("CPU%u enabling wrong PMNC counter %d\n",
886 smp_processor_id(), idx);
887 return -EINVAL;
888 }
889
890 counter = ARMV8_IDX_TO_COUNTER(idx);
891 asm volatile("msr pmcntenset_el0, %0" :: "r" (BIT(counter))); 318 asm volatile("msr pmcntenset_el0, %0" :: "r" (BIT(counter)));
892 return idx; 319 return idx;
893} 320}
894 321
895static inline int armv8pmu_disable_counter(int idx) 322static inline int armv8pmu_disable_counter(int idx)
896{ 323{
897 u32 counter; 324 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
898
899 if (!armv8pmu_counter_valid(idx)) {
900 pr_err("CPU%u disabling wrong PMNC counter %d\n",
901 smp_processor_id(), idx);
902 return -EINVAL;
903 }
904
905 counter = ARMV8_IDX_TO_COUNTER(idx);
906 asm volatile("msr pmcntenclr_el0, %0" :: "r" (BIT(counter))); 325 asm volatile("msr pmcntenclr_el0, %0" :: "r" (BIT(counter)));
907 return idx; 326 return idx;
908} 327}
909 328
910static inline int armv8pmu_enable_intens(int idx) 329static inline int armv8pmu_enable_intens(int idx)
911{ 330{
912 u32 counter; 331 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
913
914 if (!armv8pmu_counter_valid(idx)) {
915 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
916 smp_processor_id(), idx);
917 return -EINVAL;
918 }
919
920 counter = ARMV8_IDX_TO_COUNTER(idx);
921 asm volatile("msr pmintenset_el1, %0" :: "r" (BIT(counter))); 332 asm volatile("msr pmintenset_el1, %0" :: "r" (BIT(counter)));
922 return idx; 333 return idx;
923} 334}
924 335
925static inline int armv8pmu_disable_intens(int idx) 336static inline int armv8pmu_disable_intens(int idx)
926{ 337{
927 u32 counter; 338 u32 counter = ARMV8_IDX_TO_COUNTER(idx);
928
929 if (!armv8pmu_counter_valid(idx)) {
930 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
931 smp_processor_id(), idx);
932 return -EINVAL;
933 }
934
935 counter = ARMV8_IDX_TO_COUNTER(idx);
936 asm volatile("msr pmintenclr_el1, %0" :: "r" (BIT(counter))); 339 asm volatile("msr pmintenclr_el1, %0" :: "r" (BIT(counter)));
937 isb(); 340 isb();
938 /* Clear the overflow flag in case an interrupt is pending. */ 341 /* Clear the overflow flag in case an interrupt is pending. */
939 asm volatile("msr pmovsclr_el0, %0" :: "r" (BIT(counter))); 342 asm volatile("msr pmovsclr_el0, %0" :: "r" (BIT(counter)));
940 isb(); 343 isb();
344
941 return idx; 345 return idx;
942} 346}
943 347
@@ -955,10 +359,13 @@ static inline u32 armv8pmu_getreset_flags(void)
955 return value; 359 return value;
956} 360}
957 361
958static void armv8pmu_enable_event(struct hw_perf_event *hwc, int idx) 362static void armv8pmu_enable_event(struct perf_event *event)
959{ 363{
960 unsigned long flags; 364 unsigned long flags;
961 struct pmu_hw_events *events = cpu_pmu->get_hw_events(); 365 struct hw_perf_event *hwc = &event->hw;
366 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
367 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
368 int idx = hwc->idx;
962 369
963 /* 370 /*
964 * Enable counter and interrupt, and set the counter to count 371 * Enable counter and interrupt, and set the counter to count
@@ -989,10 +396,13 @@ static void armv8pmu_enable_event(struct hw_perf_event *hwc, int idx)
989 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 396 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
990} 397}
991 398
992static void armv8pmu_disable_event(struct hw_perf_event *hwc, int idx) 399static void armv8pmu_disable_event(struct perf_event *event)
993{ 400{
994 unsigned long flags; 401 unsigned long flags;
995 struct pmu_hw_events *events = cpu_pmu->get_hw_events(); 402 struct hw_perf_event *hwc = &event->hw;
403 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
404 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
405 int idx = hwc->idx;
996 406
997 /* 407 /*
998 * Disable counter and interrupt 408 * Disable counter and interrupt
@@ -1016,7 +426,8 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
1016{ 426{
1017 u32 pmovsr; 427 u32 pmovsr;
1018 struct perf_sample_data data; 428 struct perf_sample_data data;
1019 struct pmu_hw_events *cpuc; 429 struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
430 struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
1020 struct pt_regs *regs; 431 struct pt_regs *regs;
1021 int idx; 432 int idx;
1022 433
@@ -1036,7 +447,6 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
1036 */ 447 */
1037 regs = get_irq_regs(); 448 regs = get_irq_regs();
1038 449
1039 cpuc = this_cpu_ptr(&cpu_hw_events);
1040 for (idx = 0; idx < cpu_pmu->num_events; ++idx) { 450 for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
1041 struct perf_event *event = cpuc->events[idx]; 451 struct perf_event *event = cpuc->events[idx];
1042 struct hw_perf_event *hwc; 452 struct hw_perf_event *hwc;
@@ -1053,13 +463,13 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
1053 continue; 463 continue;
1054 464
1055 hwc = &event->hw; 465 hwc = &event->hw;
1056 armpmu_event_update(event, hwc, idx); 466 armpmu_event_update(event);
1057 perf_sample_data_init(&data, 0, hwc->last_period); 467 perf_sample_data_init(&data, 0, hwc->last_period);
1058 if (!armpmu_event_set_period(event, hwc, idx)) 468 if (!armpmu_event_set_period(event))
1059 continue; 469 continue;
1060 470
1061 if (perf_event_overflow(event, &data, regs)) 471 if (perf_event_overflow(event, &data, regs))
1062 cpu_pmu->disable(hwc, idx); 472 cpu_pmu->disable(event);
1063 } 473 }
1064 474
1065 /* 475 /*
@@ -1074,10 +484,10 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
1074 return IRQ_HANDLED; 484 return IRQ_HANDLED;
1075} 485}
1076 486
1077static void armv8pmu_start(void) 487static void armv8pmu_start(struct arm_pmu *cpu_pmu)
1078{ 488{
1079 unsigned long flags; 489 unsigned long flags;
1080 struct pmu_hw_events *events = cpu_pmu->get_hw_events(); 490 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1081 491
1082 raw_spin_lock_irqsave(&events->pmu_lock, flags); 492 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1083 /* Enable all counters */ 493 /* Enable all counters */
@@ -1085,10 +495,10 @@ static void armv8pmu_start(void)
1085 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 495 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1086} 496}
1087 497
1088static void armv8pmu_stop(void) 498static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
1089{ 499{
1090 unsigned long flags; 500 unsigned long flags;
1091 struct pmu_hw_events *events = cpu_pmu->get_hw_events(); 501 struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1092 502
1093 raw_spin_lock_irqsave(&events->pmu_lock, flags); 503 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1094 /* Disable all counters */ 504 /* Disable all counters */
@@ -1097,10 +507,12 @@ static void armv8pmu_stop(void)
1097} 507}
1098 508
1099static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, 509static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
1100 struct hw_perf_event *event) 510 struct perf_event *event)
1101{ 511{
1102 int idx; 512 int idx;
1103 unsigned long evtype = event->config_base & ARMV8_EVTYPE_EVENT; 513 struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
514 struct hw_perf_event *hwc = &event->hw;
515 unsigned long evtype = hwc->config_base & ARMV8_EVTYPE_EVENT;
1104 516
1105 /* Always place a cycle counter into the cycle counter. */ 517 /* Always place a cycle counter into the cycle counter. */
1106 if (evtype == ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES) { 518 if (evtype == ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES) {
@@ -1151,11 +563,14 @@ static int armv8pmu_set_event_filter(struct hw_perf_event *event,
1151 563
1152static void armv8pmu_reset(void *info) 564static void armv8pmu_reset(void *info)
1153{ 565{
566 struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
1154 u32 idx, nb_cnt = cpu_pmu->num_events; 567 u32 idx, nb_cnt = cpu_pmu->num_events;
1155 568
1156 /* The counter and interrupt enable registers are unknown at reset. */ 569 /* The counter and interrupt enable registers are unknown at reset. */
1157 for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) 570 for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1158 armv8pmu_disable_event(NULL, idx); 571 armv8pmu_disable_counter(idx);
572 armv8pmu_disable_intens(idx);
573 }
1159 574
1160 /* Initialize & Reset PMNC: C and P bits. */ 575 /* Initialize & Reset PMNC: C and P bits. */
1161 armv8pmu_pmcr_write(ARMV8_PMCR_P | ARMV8_PMCR_C); 576 armv8pmu_pmcr_write(ARMV8_PMCR_P | ARMV8_PMCR_C);
@@ -1166,169 +581,104 @@ static void armv8pmu_reset(void *info)
1166 581
1167static int armv8_pmuv3_map_event(struct perf_event *event) 582static int armv8_pmuv3_map_event(struct perf_event *event)
1168{ 583{
1169 return map_cpu_event(event, &armv8_pmuv3_perf_map, 584 return armpmu_map_event(event, &armv8_pmuv3_perf_map,
1170 &armv8_pmuv3_perf_cache_map, 585 &armv8_pmuv3_perf_cache_map,
1171 ARMV8_EVTYPE_EVENT); 586 ARMV8_EVTYPE_EVENT);
1172} 587}
1173 588
1174static struct arm_pmu armv8pmu = { 589static int armv8_a53_map_event(struct perf_event *event)
1175 .handle_irq = armv8pmu_handle_irq, 590{
1176 .enable = armv8pmu_enable_event, 591 return armpmu_map_event(event, &armv8_a53_perf_map,
1177 .disable = armv8pmu_disable_event, 592 &armv8_a53_perf_cache_map,
1178 .read_counter = armv8pmu_read_counter, 593 ARMV8_EVTYPE_EVENT);
1179 .write_counter = armv8pmu_write_counter, 594}
1180 .get_event_idx = armv8pmu_get_event_idx,
1181 .start = armv8pmu_start,
1182 .stop = armv8pmu_stop,
1183 .reset = armv8pmu_reset,
1184 .max_period = (1LLU << 32) - 1,
1185};
1186 595
1187static u32 __init armv8pmu_read_num_pmnc_events(void) 596static int armv8_a57_map_event(struct perf_event *event)
1188{ 597{
1189 u32 nb_cnt; 598 return armpmu_map_event(event, &armv8_a57_perf_map,
599 &armv8_a57_perf_cache_map,
600 ARMV8_EVTYPE_EVENT);
601}
602
603static void armv8pmu_read_num_pmnc_events(void *info)
604{
605 int *nb_cnt = info;
1190 606
1191 /* Read the nb of CNTx counters supported from PMNC */ 607 /* Read the nb of CNTx counters supported from PMNC */
1192 nb_cnt = (armv8pmu_pmcr_read() >> ARMV8_PMCR_N_SHIFT) & ARMV8_PMCR_N_MASK; 608 *nb_cnt = (armv8pmu_pmcr_read() >> ARMV8_PMCR_N_SHIFT) & ARMV8_PMCR_N_MASK;
1193 609
1194 /* Add the CPU cycles counter and return */ 610 /* Add the CPU cycles counter */
1195 return nb_cnt + 1; 611 *nb_cnt += 1;
1196} 612}
1197 613
1198static struct arm_pmu *__init armv8_pmuv3_pmu_init(void) 614static int armv8pmu_probe_num_events(struct arm_pmu *arm_pmu)
1199{ 615{
1200 armv8pmu.name = "arm/armv8-pmuv3"; 616 return smp_call_function_any(&arm_pmu->supported_cpus,
1201 armv8pmu.map_event = armv8_pmuv3_map_event; 617 armv8pmu_read_num_pmnc_events,
1202 armv8pmu.num_events = armv8pmu_read_num_pmnc_events(); 618 &arm_pmu->num_events, 1);
1203 armv8pmu.set_event_filter = armv8pmu_set_event_filter;
1204 return &armv8pmu;
1205} 619}
1206 620
1207/* 621static void armv8_pmu_init(struct arm_pmu *cpu_pmu)
1208 * Ensure the PMU has sane values out of reset.
1209 * This requires SMP to be available, so exists as a separate initcall.
1210 */
1211static int __init
1212cpu_pmu_reset(void)
1213{ 622{
1214 if (cpu_pmu && cpu_pmu->reset) 623 cpu_pmu->handle_irq = armv8pmu_handle_irq,
1215 return on_each_cpu(cpu_pmu->reset, NULL, 1); 624 cpu_pmu->enable = armv8pmu_enable_event,
1216 return 0; 625 cpu_pmu->disable = armv8pmu_disable_event,
626 cpu_pmu->read_counter = armv8pmu_read_counter,
627 cpu_pmu->write_counter = armv8pmu_write_counter,
628 cpu_pmu->get_event_idx = armv8pmu_get_event_idx,
629 cpu_pmu->start = armv8pmu_start,
630 cpu_pmu->stop = armv8pmu_stop,
631 cpu_pmu->reset = armv8pmu_reset,
632 cpu_pmu->max_period = (1LLU << 32) - 1,
633 cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
1217} 634}
1218arch_initcall(cpu_pmu_reset);
1219
1220/*
1221 * PMU platform driver and devicetree bindings.
1222 */
1223static const struct of_device_id armpmu_of_device_ids[] = {
1224 {.compatible = "arm,armv8-pmuv3"},
1225 {},
1226};
1227 635
1228static int armpmu_device_probe(struct platform_device *pdev) 636static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)
1229{ 637{
1230 int i, irq, *irqs; 638 armv8_pmu_init(cpu_pmu);
1231 639 cpu_pmu->name = "armv8_pmuv3";
1232 if (!cpu_pmu) 640 cpu_pmu->map_event = armv8_pmuv3_map_event;
1233 return -ENODEV; 641 return armv8pmu_probe_num_events(cpu_pmu);
1234
1235 /* Don't bother with PPIs; they're already affine */
1236 irq = platform_get_irq(pdev, 0);
1237 if (irq >= 0 && irq_is_percpu(irq))
1238 goto out;
1239
1240 irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);
1241 if (!irqs)
1242 return -ENOMEM;
1243
1244 for (i = 0; i < pdev->num_resources; ++i) {
1245 struct device_node *dn;
1246 int cpu;
1247
1248 dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity",
1249 i);
1250 if (!dn) {
1251 pr_warn("Failed to parse %s/interrupt-affinity[%d]\n",
1252 of_node_full_name(pdev->dev.of_node), i);
1253 break;
1254 }
1255
1256 for_each_possible_cpu(cpu)
1257 if (dn == of_cpu_device_node_get(cpu))
1258 break;
1259
1260 if (cpu >= nr_cpu_ids) {
1261 pr_warn("Failed to find logical CPU for %s\n",
1262 dn->name);
1263 of_node_put(dn);
1264 break;
1265 }
1266 of_node_put(dn);
1267
1268 irqs[i] = cpu;
1269 }
1270
1271 if (i == pdev->num_resources)
1272 cpu_pmu->irq_affinity = irqs;
1273 else
1274 kfree(irqs);
1275
1276out:
1277 cpu_pmu->plat_device = pdev;
1278 return 0;
1279} 642}
1280 643
1281static struct platform_driver armpmu_driver = { 644static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
1282 .driver = {
1283 .name = "arm-pmu",
1284 .of_match_table = armpmu_of_device_ids,
1285 },
1286 .probe = armpmu_device_probe,
1287};
1288
1289static int __init register_pmu_driver(void)
1290{ 645{
1291 return platform_driver_register(&armpmu_driver); 646 armv8_pmu_init(cpu_pmu);
647 cpu_pmu->name = "armv8_cortex_a53";
648 cpu_pmu->map_event = armv8_a53_map_event;
649 return armv8pmu_probe_num_events(cpu_pmu);
1292} 650}
1293device_initcall(register_pmu_driver);
1294 651
1295static struct pmu_hw_events *armpmu_get_cpu_events(void) 652static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
1296{ 653{
1297 return this_cpu_ptr(&cpu_hw_events); 654 armv8_pmu_init(cpu_pmu);
655 cpu_pmu->name = "armv8_cortex_a57";
656 cpu_pmu->map_event = armv8_a57_map_event;
657 return armv8pmu_probe_num_events(cpu_pmu);
1298} 658}
1299 659
1300static void __init cpu_pmu_init(struct arm_pmu *armpmu) 660static const struct of_device_id armv8_pmu_of_device_ids[] = {
1301{ 661 {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init},
1302 int cpu; 662 {.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init},
1303 for_each_possible_cpu(cpu) { 663 {.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init},
1304 struct pmu_hw_events *events = &per_cpu(cpu_hw_events, cpu); 664 {},
1305 events->events = per_cpu(hw_events, cpu); 665};
1306 events->used_mask = per_cpu(used_mask, cpu);
1307 raw_spin_lock_init(&events->pmu_lock);
1308 }
1309 armpmu->get_hw_events = armpmu_get_cpu_events;
1310}
1311 666
1312static int __init init_hw_perf_events(void) 667static int armv8_pmu_device_probe(struct platform_device *pdev)
1313{ 668{
1314 u64 dfr = read_cpuid(ID_AA64DFR0_EL1); 669 return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL);
1315 670}
1316 switch ((dfr >> 8) & 0xf) {
1317 case 0x1: /* PMUv3 */
1318 cpu_pmu = armv8_pmuv3_pmu_init();
1319 break;
1320 }
1321 671
1322 if (cpu_pmu) { 672static struct platform_driver armv8_pmu_driver = {
1323 pr_info("enabled with %s PMU driver, %d counters available\n", 673 .driver = {
1324 cpu_pmu->name, cpu_pmu->num_events); 674 .name = "armv8-pmu",
1325 cpu_pmu_init(cpu_pmu); 675 .of_match_table = armv8_pmu_of_device_ids,
1326 armpmu_register(cpu_pmu, "cpu", PERF_TYPE_RAW); 676 },
1327 } else { 677 .probe = armv8_pmu_device_probe,
1328 pr_info("no hardware support available\n"); 678};
1329 }
1330 679
1331 return 0; 680static int __init register_armv8_pmu_driver(void)
681{
682 return platform_driver_register(&armv8_pmu_driver);
1332} 683}
1333early_initcall(init_hw_perf_events); 684device_initcall(register_armv8_pmu_driver);
1334
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 223b093c9440..f75b540bc3b4 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -44,6 +44,7 @@
44#include <linux/hw_breakpoint.h> 44#include <linux/hw_breakpoint.h>
45#include <linux/personality.h> 45#include <linux/personality.h>
46#include <linux/notifier.h> 46#include <linux/notifier.h>
47#include <trace/events/power.h>
47 48
48#include <asm/compat.h> 49#include <asm/compat.h>
49#include <asm/cacheflush.h> 50#include <asm/cacheflush.h>
@@ -75,8 +76,10 @@ void arch_cpu_idle(void)
75 * This should do all the clock switching and wait for interrupt 76 * This should do all the clock switching and wait for interrupt
76 * tricks 77 * tricks
77 */ 78 */
79 trace_cpu_idle_rcuidle(1, smp_processor_id());
78 cpu_do_idle(); 80 cpu_do_idle();
79 local_irq_enable(); 81 local_irq_enable();
82 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
80} 83}
81 84
82#ifdef CONFIG_HOTPLUG_CPU 85#ifdef CONFIG_HOTPLUG_CPU
diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c
index aa94a88f6279..f67f35b6edb1 100644
--- a/arch/arm64/kernel/psci.c
+++ b/arch/arm64/kernel/psci.c
@@ -30,20 +30,6 @@
30#include <asm/smp_plat.h> 30#include <asm/smp_plat.h>
31#include <asm/suspend.h> 31#include <asm/suspend.h>
32 32
33static bool psci_power_state_loses_context(u32 state)
34{
35 return state & PSCI_0_2_POWER_STATE_TYPE_MASK;
36}
37
38static bool psci_power_state_is_valid(u32 state)
39{
40 const u32 valid_mask = PSCI_0_2_POWER_STATE_ID_MASK |
41 PSCI_0_2_POWER_STATE_TYPE_MASK |
42 PSCI_0_2_POWER_STATE_AFFL_MASK;
43
44 return !(state & ~valid_mask);
45}
46
47static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state); 33static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state);
48 34
49static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) 35static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu)
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 232247945b1c..8119479147db 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -28,7 +28,6 @@
28#include <linux/console.h> 28#include <linux/console.h>
29#include <linux/cache.h> 29#include <linux/cache.h>
30#include <linux/bootmem.h> 30#include <linux/bootmem.h>
31#include <linux/seq_file.h>
32#include <linux/screen_info.h> 31#include <linux/screen_info.h>
33#include <linux/init.h> 32#include <linux/init.h>
34#include <linux/kexec.h> 33#include <linux/kexec.h>
@@ -44,7 +43,6 @@
44#include <linux/of_fdt.h> 43#include <linux/of_fdt.h>
45#include <linux/of_platform.h> 44#include <linux/of_platform.h>
46#include <linux/efi.h> 45#include <linux/efi.h>
47#include <linux/personality.h>
48#include <linux/psci.h> 46#include <linux/psci.h>
49 47
50#include <asm/acpi.h> 48#include <asm/acpi.h>
@@ -54,6 +52,7 @@
54#include <asm/elf.h> 52#include <asm/elf.h>
55#include <asm/cpufeature.h> 53#include <asm/cpufeature.h>
56#include <asm/cpu_ops.h> 54#include <asm/cpu_ops.h>
55#include <asm/kasan.h>
57#include <asm/sections.h> 56#include <asm/sections.h>
58#include <asm/setup.h> 57#include <asm/setup.h>
59#include <asm/smp_plat.h> 58#include <asm/smp_plat.h>
@@ -64,23 +63,6 @@
64#include <asm/efi.h> 63#include <asm/efi.h>
65#include <asm/xen/hypervisor.h> 64#include <asm/xen/hypervisor.h>
66 65
67unsigned long elf_hwcap __read_mostly;
68EXPORT_SYMBOL_GPL(elf_hwcap);
69
70#ifdef CONFIG_COMPAT
71#define COMPAT_ELF_HWCAP_DEFAULT \
72 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
73 COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
74 COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
75 COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
76 COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
77 COMPAT_HWCAP_LPAE)
78unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
79unsigned int compat_elf_hwcap2 __read_mostly;
80#endif
81
82DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
83
84phys_addr_t __fdt_pointer __initdata; 66phys_addr_t __fdt_pointer __initdata;
85 67
86/* 68/*
@@ -195,104 +177,6 @@ static void __init smp_build_mpidr_hash(void)
195 __flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash)); 177 __flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash));
196} 178}
197 179
198static void __init setup_processor(void)
199{
200 u64 features;
201 s64 block;
202 u32 cwg;
203 int cls;
204
205 printk("CPU: AArch64 Processor [%08x] revision %d\n",
206 read_cpuid_id(), read_cpuid_id() & 15);
207
208 sprintf(init_utsname()->machine, ELF_PLATFORM);
209 elf_hwcap = 0;
210
211 cpuinfo_store_boot_cpu();
212
213 /*
214 * Check for sane CTR_EL0.CWG value.
215 */
216 cwg = cache_type_cwg();
217 cls = cache_line_size();
218 if (!cwg)
219 pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n",
220 cls);
221 if (L1_CACHE_BYTES < cls)
222 pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n",
223 L1_CACHE_BYTES, cls);
224
225 /*
226 * ID_AA64ISAR0_EL1 contains 4-bit wide signed feature blocks.
227 * The blocks we test below represent incremental functionality
228 * for non-negative values. Negative values are reserved.
229 */
230 features = read_cpuid(ID_AA64ISAR0_EL1);
231 block = cpuid_feature_extract_field(features, 4);
232 if (block > 0) {
233 switch (block) {
234 default:
235 case 2:
236 elf_hwcap |= HWCAP_PMULL;
237 case 1:
238 elf_hwcap |= HWCAP_AES;
239 case 0:
240 break;
241 }
242 }
243
244 if (cpuid_feature_extract_field(features, 8) > 0)
245 elf_hwcap |= HWCAP_SHA1;
246
247 if (cpuid_feature_extract_field(features, 12) > 0)
248 elf_hwcap |= HWCAP_SHA2;
249
250 if (cpuid_feature_extract_field(features, 16) > 0)
251 elf_hwcap |= HWCAP_CRC32;
252
253 block = cpuid_feature_extract_field(features, 20);
254 if (block > 0) {
255 switch (block) {
256 default:
257 case 2:
258 elf_hwcap |= HWCAP_ATOMICS;
259 case 1:
260 /* RESERVED */
261 case 0:
262 break;
263 }
264 }
265
266#ifdef CONFIG_COMPAT
267 /*
268 * ID_ISAR5_EL1 carries similar information as above, but pertaining to
269 * the AArch32 32-bit execution state.
270 */
271 features = read_cpuid(ID_ISAR5_EL1);
272 block = cpuid_feature_extract_field(features, 4);
273 if (block > 0) {
274 switch (block) {
275 default:
276 case 2:
277 compat_elf_hwcap2 |= COMPAT_HWCAP2_PMULL;
278 case 1:
279 compat_elf_hwcap2 |= COMPAT_HWCAP2_AES;
280 case 0:
281 break;
282 }
283 }
284
285 if (cpuid_feature_extract_field(features, 8) > 0)
286 compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA1;
287
288 if (cpuid_feature_extract_field(features, 12) > 0)
289 compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA2;
290
291 if (cpuid_feature_extract_field(features, 16) > 0)
292 compat_elf_hwcap2 |= COMPAT_HWCAP2_CRC32;
293#endif
294}
295
296static void __init setup_machine_fdt(phys_addr_t dt_phys) 180static void __init setup_machine_fdt(phys_addr_t dt_phys)
297{ 181{
298 void *dt_virt = fixmap_remap_fdt(dt_phys); 182 void *dt_virt = fixmap_remap_fdt(dt_phys);
@@ -406,8 +290,9 @@ u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
406 290
407void __init setup_arch(char **cmdline_p) 291void __init setup_arch(char **cmdline_p)
408{ 292{
409 setup_processor(); 293 pr_info("Boot CPU: AArch64 Processor [%08x]\n", read_cpuid_id());
410 294
295 sprintf(init_utsname()->machine, ELF_PLATFORM);
411 init_mm.start_code = (unsigned long) _text; 296 init_mm.start_code = (unsigned long) _text;
412 init_mm.end_code = (unsigned long) _etext; 297 init_mm.end_code = (unsigned long) _etext;
413 init_mm.end_data = (unsigned long) _edata; 298 init_mm.end_data = (unsigned long) _edata;
@@ -436,6 +321,9 @@ void __init setup_arch(char **cmdline_p)
436 321
437 paging_init(); 322 paging_init();
438 relocate_initrd(); 323 relocate_initrd();
324
325 kasan_init();
326
439 request_standard_resources(); 327 request_standard_resources();
440 328
441 early_ioremap_reset(); 329 early_ioremap_reset();
@@ -493,124 +381,3 @@ static int __init topology_init(void)
493 return 0; 381 return 0;
494} 382}
495subsys_initcall(topology_init); 383subsys_initcall(topology_init);
496
497static const char *hwcap_str[] = {
498 "fp",
499 "asimd",
500 "evtstrm",
501 "aes",
502 "pmull",
503 "sha1",
504 "sha2",
505 "crc32",
506 "atomics",
507 NULL
508};
509
510#ifdef CONFIG_COMPAT
511static const char *compat_hwcap_str[] = {
512 "swp",
513 "half",
514 "thumb",
515 "26bit",
516 "fastmult",
517 "fpa",
518 "vfp",
519 "edsp",
520 "java",
521 "iwmmxt",
522 "crunch",
523 "thumbee",
524 "neon",
525 "vfpv3",
526 "vfpv3d16",
527 "tls",
528 "vfpv4",
529 "idiva",
530 "idivt",
531 "vfpd32",
532 "lpae",
533 "evtstrm"
534};
535
536static const char *compat_hwcap2_str[] = {
537 "aes",
538 "pmull",
539 "sha1",
540 "sha2",
541 "crc32",
542 NULL
543};
544#endif /* CONFIG_COMPAT */
545
546static int c_show(struct seq_file *m, void *v)
547{
548 int i, j;
549
550 for_each_online_cpu(i) {
551 struct cpuinfo_arm64 *cpuinfo = &per_cpu(cpu_data, i);
552 u32 midr = cpuinfo->reg_midr;
553
554 /*
555 * glibc reads /proc/cpuinfo to determine the number of
556 * online processors, looking for lines beginning with
557 * "processor". Give glibc what it expects.
558 */
559 seq_printf(m, "processor\t: %d\n", i);
560
561 /*
562 * Dump out the common processor features in a single line.
563 * Userspace should read the hwcaps with getauxval(AT_HWCAP)
564 * rather than attempting to parse this, but there's a body of
565 * software which does already (at least for 32-bit).
566 */
567 seq_puts(m, "Features\t:");
568 if (personality(current->personality) == PER_LINUX32) {
569#ifdef CONFIG_COMPAT
570 for (j = 0; compat_hwcap_str[j]; j++)
571 if (compat_elf_hwcap & (1 << j))
572 seq_printf(m, " %s", compat_hwcap_str[j]);
573
574 for (j = 0; compat_hwcap2_str[j]; j++)
575 if (compat_elf_hwcap2 & (1 << j))
576 seq_printf(m, " %s", compat_hwcap2_str[j]);
577#endif /* CONFIG_COMPAT */
578 } else {
579 for (j = 0; hwcap_str[j]; j++)
580 if (elf_hwcap & (1 << j))
581 seq_printf(m, " %s", hwcap_str[j]);
582 }
583 seq_puts(m, "\n");
584
585 seq_printf(m, "CPU implementer\t: 0x%02x\n",
586 MIDR_IMPLEMENTOR(midr));
587 seq_printf(m, "CPU architecture: 8\n");
588 seq_printf(m, "CPU variant\t: 0x%x\n", MIDR_VARIANT(midr));
589 seq_printf(m, "CPU part\t: 0x%03x\n", MIDR_PARTNUM(midr));
590 seq_printf(m, "CPU revision\t: %d\n\n", MIDR_REVISION(midr));
591 }
592
593 return 0;
594}
595
596static void *c_start(struct seq_file *m, loff_t *pos)
597{
598 return *pos < 1 ? (void *)1 : NULL;
599}
600
601static void *c_next(struct seq_file *m, void *v, loff_t *pos)
602{
603 ++*pos;
604 return NULL;
605}
606
607static void c_stop(struct seq_file *m, void *v)
608{
609}
610
611const struct seq_operations cpuinfo_op = {
612 .start = c_start,
613 .next = c_next,
614 .stop = c_stop,
615 .show = c_show
616};
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index dbdaacddd9a5..b1adc51b2c2e 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -142,22 +142,27 @@ asmlinkage void secondary_start_kernel(void)
142 */ 142 */
143 atomic_inc(&mm->mm_count); 143 atomic_inc(&mm->mm_count);
144 current->active_mm = mm; 144 current->active_mm = mm;
145 cpumask_set_cpu(cpu, mm_cpumask(mm));
146 145
147 set_my_cpu_offset(per_cpu_offset(smp_processor_id())); 146 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
148 printk("CPU%u: Booted secondary processor\n", cpu);
149 147
150 /* 148 /*
151 * TTBR0 is only used for the identity mapping at this stage. Make it 149 * TTBR0 is only used for the identity mapping at this stage. Make it
152 * point to zero page to avoid speculatively fetching new entries. 150 * point to zero page to avoid speculatively fetching new entries.
153 */ 151 */
154 cpu_set_reserved_ttbr0(); 152 cpu_set_reserved_ttbr0();
155 flush_tlb_all(); 153 local_flush_tlb_all();
156 cpu_set_default_tcr_t0sz(); 154 cpu_set_default_tcr_t0sz();
157 155
158 preempt_disable(); 156 preempt_disable();
159 trace_hardirqs_off(); 157 trace_hardirqs_off();
160 158
159 /*
160 * If the system has established the capabilities, make sure
161 * this CPU ticks all of those. If it doesn't, the CPU will
162 * fail to come online.
163 */
164 verify_local_cpu_capabilities();
165
161 if (cpu_ops[cpu]->cpu_postboot) 166 if (cpu_ops[cpu]->cpu_postboot)
162 cpu_ops[cpu]->cpu_postboot(); 167 cpu_ops[cpu]->cpu_postboot();
163 168
@@ -178,6 +183,8 @@ asmlinkage void secondary_start_kernel(void)
178 * the CPU migration code to notice that the CPU is online 183 * the CPU migration code to notice that the CPU is online
179 * before we continue. 184 * before we continue.
180 */ 185 */
186 pr_info("CPU%u: Booted secondary processor [%08x]\n",
187 cpu, read_cpuid_id());
181 set_cpu_online(cpu, true); 188 set_cpu_online(cpu, true);
182 complete(&cpu_running); 189 complete(&cpu_running);
183 190
@@ -232,12 +239,7 @@ int __cpu_disable(void)
232 /* 239 /*
233 * OK - migrate IRQs away from this CPU 240 * OK - migrate IRQs away from this CPU
234 */ 241 */
235 migrate_irqs(); 242 irq_migrate_all_off_this_cpu();
236
237 /*
238 * Remove this CPU from the vm mask set of all processes.
239 */
240 clear_tasks_mm_cpumask(cpu);
241 243
242 return 0; 244 return 0;
243} 245}
@@ -325,12 +327,14 @@ static void __init hyp_mode_check(void)
325void __init smp_cpus_done(unsigned int max_cpus) 327void __init smp_cpus_done(unsigned int max_cpus)
326{ 328{
327 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); 329 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
330 setup_cpu_features();
328 hyp_mode_check(); 331 hyp_mode_check();
329 apply_alternatives_all(); 332 apply_alternatives_all();
330} 333}
331 334
332void __init smp_prepare_boot_cpu(void) 335void __init smp_prepare_boot_cpu(void)
333{ 336{
337 cpuinfo_store_boot_cpu();
334 set_my_cpu_offset(per_cpu_offset(smp_processor_id())); 338 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
335} 339}
336 340
@@ -469,7 +473,7 @@ acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header,
469 * cpu logical map array containing MPIDR values related to logical 473 * cpu logical map array containing MPIDR values related to logical
470 * cpus. Assumes that cpu_logical_map(0) has already been initialized. 474 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
471 */ 475 */
472void __init of_parse_and_init_cpus(void) 476static void __init of_parse_and_init_cpus(void)
473{ 477{
474 struct device_node *dn = NULL; 478 struct device_node *dn = NULL;
475 479
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index 407991bf79f5..ccb6078ed9f2 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -48,11 +48,7 @@ int notrace unwind_frame(struct stackframe *frame)
48 48
49 frame->sp = fp + 0x10; 49 frame->sp = fp + 0x10;
50 frame->fp = *(unsigned long *)(fp); 50 frame->fp = *(unsigned long *)(fp);
51 /* 51 frame->pc = *(unsigned long *)(fp + 8);
52 * -4 here because we care about the PC at time of bl,
53 * not where the return will go.
54 */
55 frame->pc = *(unsigned long *)(fp + 8) - 4;
56 52
57 return 0; 53 return 0;
58} 54}
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 8297d502217e..1095aa483a1c 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -1,3 +1,4 @@
1#include <linux/ftrace.h>
1#include <linux/percpu.h> 2#include <linux/percpu.h>
2#include <linux/slab.h> 3#include <linux/slab.h>
3#include <asm/cacheflush.h> 4#include <asm/cacheflush.h>
@@ -41,7 +42,7 @@ void notrace __cpu_suspend_save(struct cpu_suspend_ctx *ptr,
41 * time the notifier runs debug exceptions might have been enabled already, 42 * time the notifier runs debug exceptions might have been enabled already,
42 * with HW breakpoints registers content still in an unknown state. 43 * with HW breakpoints registers content still in an unknown state.
43 */ 44 */
44void (*hw_breakpoint_restore)(void *); 45static void (*hw_breakpoint_restore)(void *);
45void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) 46void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
46{ 47{
47 /* Prevent multiple restore hook initializations */ 48 /* Prevent multiple restore hook initializations */
@@ -71,6 +72,13 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
71 local_dbg_save(flags); 72 local_dbg_save(flags);
72 73
73 /* 74 /*
75 * Function graph tracer state gets incosistent when the kernel
76 * calls functions that never return (aka suspend finishers) hence
77 * disable graph tracing during their execution.
78 */
79 pause_graph_tracing();
80
81 /*
74 * mm context saved on the stack, it will be restored when 82 * mm context saved on the stack, it will be restored when
75 * the cpu comes out of reset through the identity mapped 83 * the cpu comes out of reset through the identity mapped
76 * page tables, so that the thread address space is properly 84 * page tables, so that the thread address space is properly
@@ -80,17 +88,21 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
80 if (ret == 0) { 88 if (ret == 0) {
81 /* 89 /*
82 * We are resuming from reset with TTBR0_EL1 set to the 90 * We are resuming from reset with TTBR0_EL1 set to the
83 * idmap to enable the MMU; restore the active_mm mappings in 91 * idmap to enable the MMU; set the TTBR0 to the reserved
84 * TTBR0_EL1 unless the active_mm == &init_mm, in which case 92 * page tables to prevent speculative TLB allocations, flush
85 * the thread entered cpu_suspend with TTBR0_EL1 set to 93 * the local tlb and set the default tcr_el1.t0sz so that
86 * reserved TTBR0 page tables and should be restored as such. 94 * the TTBR0 address space set-up is properly restored.
95 * If the current active_mm != &init_mm we entered cpu_suspend
96 * with mappings in TTBR0 that must be restored, so we switch
97 * them back to complete the address space configuration
98 * restoration before returning.
87 */ 99 */
88 if (mm == &init_mm) 100 cpu_set_reserved_ttbr0();
89 cpu_set_reserved_ttbr0(); 101 local_flush_tlb_all();
90 else 102 cpu_set_default_tcr_t0sz();
91 cpu_switch_mm(mm->pgd, mm);
92 103
93 flush_tlb_all(); 104 if (mm != &init_mm)
105 cpu_switch_mm(mm->pgd, mm);
94 106
95 /* 107 /*
96 * Restore per-cpu offset before any kernel 108 * Restore per-cpu offset before any kernel
@@ -107,6 +119,8 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
107 hw_breakpoint_restore(NULL); 119 hw_breakpoint_restore(NULL);
108 } 120 }
109 121
122 unpause_graph_tracing();
123
110 /* 124 /*
111 * Restore pstate flags. OS lock and mdscr have been already 125 * Restore pstate flags. OS lock and mdscr have been already
112 * restored, so from this point onwards, debugging is fully 126 * restored, so from this point onwards, debugging is fully
diff --git a/arch/arm64/kernel/time.c b/arch/arm64/kernel/time.c
index 149151fb42bb..13339b6ffc1a 100644
--- a/arch/arm64/kernel/time.c
+++ b/arch/arm64/kernel/time.c
@@ -67,16 +67,10 @@ void __init time_init(void)
67 u32 arch_timer_rate; 67 u32 arch_timer_rate;
68 68
69 of_clk_init(NULL); 69 of_clk_init(NULL);
70 clocksource_of_init(); 70 clocksource_probe();
71 71
72 tick_setup_hrtimer_broadcast(); 72 tick_setup_hrtimer_broadcast();
73 73
74 /*
75 * Since ACPI or FDT will only one be available in the system,
76 * we can use acpi_generic_timer_init() here safely
77 */
78 acpi_generic_timer_init();
79
80 arch_timer_rate = arch_timer_get_rate(); 74 arch_timer_rate = arch_timer_get_rate();
81 if (!arch_timer_rate) 75 if (!arch_timer_rate)
82 panic("Unable to initialise architected timer.\n"); 76 panic("Unable to initialise architected timer.\n");
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index f93aae5e4307..e9b9b5364393 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -103,12 +103,12 @@ static void dump_mem(const char *lvl, const char *str, unsigned long bottom,
103 set_fs(fs); 103 set_fs(fs);
104} 104}
105 105
106static void dump_backtrace_entry(unsigned long where, unsigned long stack) 106static void dump_backtrace_entry(unsigned long where)
107{ 107{
108 /*
109 * Note that 'where' can have a physical address, but it's not handled.
110 */
108 print_ip_sym(where); 111 print_ip_sym(where);
109 if (in_exception_text(where))
110 dump_mem("", "Exception stack", stack,
111 stack + sizeof(struct pt_regs), false);
112} 112}
113 113
114static void dump_instr(const char *lvl, struct pt_regs *regs) 114static void dump_instr(const char *lvl, struct pt_regs *regs)
@@ -172,12 +172,17 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
172 pr_emerg("Call trace:\n"); 172 pr_emerg("Call trace:\n");
173 while (1) { 173 while (1) {
174 unsigned long where = frame.pc; 174 unsigned long where = frame.pc;
175 unsigned long stack;
175 int ret; 176 int ret;
176 177
178 dump_backtrace_entry(where);
177 ret = unwind_frame(&frame); 179 ret = unwind_frame(&frame);
178 if (ret < 0) 180 if (ret < 0)
179 break; 181 break;
180 dump_backtrace_entry(where, frame.sp); 182 stack = frame.sp;
183 if (in_exception_text(where))
184 dump_mem("", "Exception stack", stack,
185 stack + sizeof(struct pt_regs), false);
181 } 186 }
182} 187}
183 188
diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
index f6fe17d88da5..b467fd0a384b 100644
--- a/arch/arm64/kernel/vdso/Makefile
+++ b/arch/arm64/kernel/vdso/Makefile
@@ -15,6 +15,9 @@ ccflags-y := -shared -fno-common -fno-builtin
15ccflags-y += -nostdlib -Wl,-soname=linux-vdso.so.1 \ 15ccflags-y += -nostdlib -Wl,-soname=linux-vdso.so.1 \
16 $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) 16 $(call cc-ldoption, -Wl$(comma)--hash-style=sysv)
17 17
18# Disable gcov profiling for VDSO code
19GCOV_PROFILE := n
20
18# Workaround for bare-metal (ELF) toolchains that neglect to pass -shared 21# Workaround for bare-metal (ELF) toolchains that neglect to pass -shared
19# down to collect2, resulting in silent corruption of the vDSO image. 22# down to collect2, resulting in silent corruption of the vDSO image.
20ccflags-y += -Wl,-shared 23ccflags-y += -Wl,-shared
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 98073332e2d0..1ee2c3937d4e 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -5,6 +5,7 @@
5 */ 5 */
6 6
7#include <asm-generic/vmlinux.lds.h> 7#include <asm-generic/vmlinux.lds.h>
8#include <asm/kernel-pgtable.h>
8#include <asm/thread_info.h> 9#include <asm/thread_info.h>
9#include <asm/memory.h> 10#include <asm/memory.h>
10#include <asm/page.h> 11#include <asm/page.h>
@@ -60,9 +61,12 @@ PECOFF_FILE_ALIGNMENT = 0x200;
60#define PECOFF_EDATA_PADDING 61#define PECOFF_EDATA_PADDING
61#endif 62#endif
62 63
63#ifdef CONFIG_DEBUG_ALIGN_RODATA 64#if defined(CONFIG_DEBUG_ALIGN_RODATA)
64#define ALIGN_DEBUG_RO . = ALIGN(1<<SECTION_SHIFT); 65#define ALIGN_DEBUG_RO . = ALIGN(1<<SECTION_SHIFT);
65#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO 66#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO
67#elif defined(CONFIG_DEBUG_RODATA)
68#define ALIGN_DEBUG_RO . = ALIGN(1<<PAGE_SHIFT);
69#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO
66#else 70#else
67#define ALIGN_DEBUG_RO 71#define ALIGN_DEBUG_RO
68#define ALIGN_DEBUG_RO_MIN(min) . = ALIGN(min); 72#define ALIGN_DEBUG_RO_MIN(min) . = ALIGN(min);
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 5c7e920e4861..a5272c07d1cb 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -16,9 +16,13 @@ menuconfig VIRTUALIZATION
16 16
17if VIRTUALIZATION 17if VIRTUALIZATION
18 18
19config KVM_ARM_VGIC_V3
20 bool
21
19config KVM 22config KVM
20 bool "Kernel-based Virtual Machine (KVM) support" 23 bool "Kernel-based Virtual Machine (KVM) support"
21 depends on OF 24 depends on OF
25 depends on !ARM64_16K_PAGES
22 select MMU_NOTIFIER 26 select MMU_NOTIFIER
23 select PREEMPT_NOTIFIERS 27 select PREEMPT_NOTIFIERS
24 select ANON_INODES 28 select ANON_INODES
@@ -31,8 +35,11 @@ config KVM
31 select KVM_VFIO 35 select KVM_VFIO
32 select HAVE_KVM_EVENTFD 36 select HAVE_KVM_EVENTFD
33 select HAVE_KVM_IRQFD 37 select HAVE_KVM_IRQFD
38 select KVM_ARM_VGIC_V3
34 ---help--- 39 ---help---
35 Support hosting virtualized guest machines. 40 Support hosting virtualized guest machines.
41 We don't support KVM with 16K page tables yet, due to the multiple
42 levels of fake page tables.
36 43
37 If unsure, say N. 44 If unsure, say N.
38 45
@@ -41,4 +48,6 @@ config KVM_ARM_HOST
41 ---help--- 48 ---help---
42 Provides host support for ARM processors. 49 Provides host support for ARM processors.
43 50
51source drivers/vhost/Kconfig
52
44endif # VIRTUALIZATION 53endif # VIRTUALIZATION
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index e5836138ec42..86c289832272 100644
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -864,6 +864,10 @@ ENTRY(__kvm_flush_vm_context)
864ENDPROC(__kvm_flush_vm_context) 864ENDPROC(__kvm_flush_vm_context)
865 865
866__kvm_hyp_panic: 866__kvm_hyp_panic:
867 // Stash PAR_EL1 before corrupting it in __restore_sysregs
868 mrs x0, par_el1
869 push x0, xzr
870
867 // Guess the context by looking at VTTBR: 871 // Guess the context by looking at VTTBR:
868 // If zero, then we're already a host. 872 // If zero, then we're already a host.
869 // Otherwise restore a minimal host context before panicing. 873 // Otherwise restore a minimal host context before panicing.
@@ -880,6 +884,14 @@ __kvm_hyp_panic:
880 884
881 bl __restore_sysregs 885 bl __restore_sysregs
882 886
887 /*
888 * Make sure we have a valid host stack, and don't leave junk in the
889 * frame pointer that will give us a misleading host stack unwinding.
890 */
891 ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
892 msr sp_el1, x22
893 mov x29, xzr
894
8831: adr x0, __hyp_panic_str 8951: adr x0, __hyp_panic_str
884 adr x1, 2f 896 adr x1, 2f
885 ldp x2, x3, [x1] 897 ldp x2, x3, [x1]
@@ -890,7 +902,7 @@ __kvm_hyp_panic:
890 mrs x3, esr_el2 902 mrs x3, esr_el2
891 mrs x4, far_el2 903 mrs x4, far_el2
892 mrs x5, hpfar_el2 904 mrs x5, hpfar_el2
893 mrs x6, par_el1 905 pop x6, xzr // active context PAR_EL1
894 mrs x7, tpidr_el2 906 mrs x7, tpidr_el2
895 907
896 mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 908 mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
@@ -906,7 +918,7 @@ __kvm_hyp_panic:
906ENDPROC(__kvm_hyp_panic) 918ENDPROC(__kvm_hyp_panic)
907 919
908__hyp_panic_str: 920__hyp_panic_str:
909 .ascii "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0" 921 .ascii "HYP panic:\nPS:%08x PC:%016x ESR:%08x\nFAR:%016x HPFAR:%016x PAR:%016x\nVCPU:%p\n\0"
910 922
911 .align 2 923 .align 2
912 924
@@ -1007,9 +1019,15 @@ el1_trap:
1007 b.ne 1f // Not an abort we care about 1019 b.ne 1f // Not an abort we care about
1008 1020
1009 /* This is an abort. Check for permission fault */ 1021 /* This is an abort. Check for permission fault */
1022alternative_if_not ARM64_WORKAROUND_834220
1010 and x2, x1, #ESR_ELx_FSC_TYPE 1023 and x2, x1, #ESR_ELx_FSC_TYPE
1011 cmp x2, #FSC_PERM 1024 cmp x2, #FSC_PERM
1012 b.ne 1f // Not a permission fault 1025 b.ne 1f // Not a permission fault
1026alternative_else
1027 nop // Use the permission fault path to
1028 nop // check for a valid S1 translation,
1029 nop // regardless of the ESR value.
1030alternative_endif
1013 1031
1014 /* 1032 /*
1015 * Check for Stage-1 page table walk, which is guaranteed 1033 * Check for Stage-1 page table walk, which is guaranteed
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 85c57158dcd9..648112e90ed5 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -48,7 +48,7 @@ static void prepare_fault32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
48 48
49 /* Note: These now point to the banked copies */ 49 /* Note: These now point to the banked copies */
50 *vcpu_spsr(vcpu) = new_spsr_value; 50 *vcpu_spsr(vcpu) = new_spsr_value;
51 *vcpu_reg(vcpu, 14) = *vcpu_pc(vcpu) + return_offset; 51 *vcpu_reg32(vcpu, 14) = *vcpu_pc(vcpu) + return_offset;
52 52
53 /* Branch to exception vector */ 53 /* Branch to exception vector */
54 if (sctlr & (1 << 13)) 54 if (sctlr & (1 << 13))
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 91cf5350b328..f34745cb3d23 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -53,7 +53,7 @@ static bool cpu_has_32bit_el1(void)
53{ 53{
54 u64 pfr0; 54 u64 pfr0;
55 55
56 pfr0 = read_cpuid(ID_AA64PFR0_EL1); 56 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1);
57 return !!(pfr0 & 0x20); 57 return !!(pfr0 & 0x20);
58} 58}
59 59
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index d03d3af17e7e..87a64e8db04c 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -693,13 +693,13 @@ static bool trap_dbgidr(struct kvm_vcpu *vcpu,
693 if (p->is_write) { 693 if (p->is_write) {
694 return ignore_write(vcpu, p); 694 return ignore_write(vcpu, p);
695 } else { 695 } else {
696 u64 dfr = read_cpuid(ID_AA64DFR0_EL1); 696 u64 dfr = read_system_reg(SYS_ID_AA64DFR0_EL1);
697 u64 pfr = read_cpuid(ID_AA64PFR0_EL1); 697 u64 pfr = read_system_reg(SYS_ID_AA64PFR0_EL1);
698 u32 el3 = !!((pfr >> 12) & 0xf); 698 u32 el3 = !!cpuid_feature_extract_field(pfr, ID_AA64PFR0_EL3_SHIFT);
699 699
700 *vcpu_reg(vcpu, p->Rt) = ((((dfr >> 20) & 0xf) << 28) | 700 *vcpu_reg(vcpu, p->Rt) = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
701 (((dfr >> 12) & 0xf) << 24) | 701 (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
702 (((dfr >> 28) & 0xf) << 20) | 702 (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20) |
703 (6 << 16) | (el3 << 14) | (el3 << 12)); 703 (6 << 16) | (el3 << 14) | (el3 << 12));
704 return true; 704 return true;
705 } 705 }
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 1be9ef27be97..4699cd74f87e 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -18,6 +18,7 @@
18 18
19#include <asm/alternative.h> 19#include <asm/alternative.h>
20#include <asm/assembler.h> 20#include <asm/assembler.h>
21#include <asm/cache.h>
21#include <asm/cpufeature.h> 22#include <asm/cpufeature.h>
22#include <asm/sysreg.h> 23#include <asm/sysreg.h>
23 24
@@ -31,49 +32,58 @@
31 * Returns: 32 * Returns:
32 * x0 - bytes not copied 33 * x0 - bytes not copied
33 */ 34 */
35
36 .macro ldrb1 ptr, regB, val
37 USER(9998f, ldrb \ptr, [\regB], \val)
38 .endm
39
40 .macro strb1 ptr, regB, val
41 strb \ptr, [\regB], \val
42 .endm
43
44 .macro ldrh1 ptr, regB, val
45 USER(9998f, ldrh \ptr, [\regB], \val)
46 .endm
47
48 .macro strh1 ptr, regB, val
49 strh \ptr, [\regB], \val
50 .endm
51
52 .macro ldr1 ptr, regB, val
53 USER(9998f, ldr \ptr, [\regB], \val)
54 .endm
55
56 .macro str1 ptr, regB, val
57 str \ptr, [\regB], \val
58 .endm
59
60 .macro ldp1 ptr, regB, regC, val
61 USER(9998f, ldp \ptr, \regB, [\regC], \val)
62 .endm
63
64 .macro stp1 ptr, regB, regC, val
65 stp \ptr, \regB, [\regC], \val
66 .endm
67
68end .req x5
34ENTRY(__copy_from_user) 69ENTRY(__copy_from_user)
35ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ 70ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
36 CONFIG_ARM64_PAN) 71 CONFIG_ARM64_PAN)
37 add x5, x1, x2 // upper user buffer boundary 72 add end, x0, x2
38 subs x2, x2, #16 73#include "copy_template.S"
39 b.mi 1f
400:
41USER(9f, ldp x3, x4, [x1], #16)
42 subs x2, x2, #16
43 stp x3, x4, [x0], #16
44 b.pl 0b
451: adds x2, x2, #8
46 b.mi 2f
47USER(9f, ldr x3, [x1], #8 )
48 sub x2, x2, #8
49 str x3, [x0], #8
502: adds x2, x2, #4
51 b.mi 3f
52USER(9f, ldr w3, [x1], #4 )
53 sub x2, x2, #4
54 str w3, [x0], #4
553: adds x2, x2, #2
56 b.mi 4f
57USER(9f, ldrh w3, [x1], #2 )
58 sub x2, x2, #2
59 strh w3, [x0], #2
604: adds x2, x2, #1
61 b.mi 5f
62USER(9f, ldrb w3, [x1] )
63 strb w3, [x0]
645: mov x0, #0
65ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ 74ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
66 CONFIG_ARM64_PAN) 75 CONFIG_ARM64_PAN)
76 mov x0, #0 // Nothing to copy
67 ret 77 ret
68ENDPROC(__copy_from_user) 78ENDPROC(__copy_from_user)
69 79
70 .section .fixup,"ax" 80 .section .fixup,"ax"
71 .align 2 81 .align 2
729: sub x2, x5, x1 829998:
73 mov x3, x2 83 sub x0, end, dst
7410: strb wzr, [x0], #1 // zero remaining buffer space 849999:
75 subs x3, x3, #1 85 strb wzr, [dst], #1 // zero remaining buffer space
76 b.ne 10b 86 cmp dst, end
77 mov x0, x2 // bytes not copied 87 b.lo 9999b
78 ret 88 ret
79 .previous 89 .previous
diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S
index 1b94661e22b3..81c8fc93c100 100644
--- a/arch/arm64/lib/copy_in_user.S
+++ b/arch/arm64/lib/copy_in_user.S
@@ -20,6 +20,7 @@
20 20
21#include <asm/alternative.h> 21#include <asm/alternative.h>
22#include <asm/assembler.h> 22#include <asm/assembler.h>
23#include <asm/cache.h>
23#include <asm/cpufeature.h> 24#include <asm/cpufeature.h>
24#include <asm/sysreg.h> 25#include <asm/sysreg.h>
25 26
@@ -33,44 +34,52 @@
33 * Returns: 34 * Returns:
34 * x0 - bytes not copied 35 * x0 - bytes not copied
35 */ 36 */
37 .macro ldrb1 ptr, regB, val
38 USER(9998f, ldrb \ptr, [\regB], \val)
39 .endm
40
41 .macro strb1 ptr, regB, val
42 USER(9998f, strb \ptr, [\regB], \val)
43 .endm
44
45 .macro ldrh1 ptr, regB, val
46 USER(9998f, ldrh \ptr, [\regB], \val)
47 .endm
48
49 .macro strh1 ptr, regB, val
50 USER(9998f, strh \ptr, [\regB], \val)
51 .endm
52
53 .macro ldr1 ptr, regB, val
54 USER(9998f, ldr \ptr, [\regB], \val)
55 .endm
56
57 .macro str1 ptr, regB, val
58 USER(9998f, str \ptr, [\regB], \val)
59 .endm
60
61 .macro ldp1 ptr, regB, regC, val
62 USER(9998f, ldp \ptr, \regB, [\regC], \val)
63 .endm
64
65 .macro stp1 ptr, regB, regC, val
66 USER(9998f, stp \ptr, \regB, [\regC], \val)
67 .endm
68
69end .req x5
36ENTRY(__copy_in_user) 70ENTRY(__copy_in_user)
37ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ 71ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
38 CONFIG_ARM64_PAN) 72 CONFIG_ARM64_PAN)
39 add x5, x0, x2 // upper user buffer boundary 73 add end, x0, x2
40 subs x2, x2, #16 74#include "copy_template.S"
41 b.mi 1f
420:
43USER(9f, ldp x3, x4, [x1], #16)
44 subs x2, x2, #16
45USER(9f, stp x3, x4, [x0], #16)
46 b.pl 0b
471: adds x2, x2, #8
48 b.mi 2f
49USER(9f, ldr x3, [x1], #8 )
50 sub x2, x2, #8
51USER(9f, str x3, [x0], #8 )
522: adds x2, x2, #4
53 b.mi 3f
54USER(9f, ldr w3, [x1], #4 )
55 sub x2, x2, #4
56USER(9f, str w3, [x0], #4 )
573: adds x2, x2, #2
58 b.mi 4f
59USER(9f, ldrh w3, [x1], #2 )
60 sub x2, x2, #2
61USER(9f, strh w3, [x0], #2 )
624: adds x2, x2, #1
63 b.mi 5f
64USER(9f, ldrb w3, [x1] )
65USER(9f, strb w3, [x0] )
665: mov x0, #0
67ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ 75ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
68 CONFIG_ARM64_PAN) 76 CONFIG_ARM64_PAN)
77 mov x0, #0
69 ret 78 ret
70ENDPROC(__copy_in_user) 79ENDPROC(__copy_in_user)
71 80
72 .section .fixup,"ax" 81 .section .fixup,"ax"
73 .align 2 82 .align 2
749: sub x0, x5, x0 // bytes not copied 839998: sub x0, end, dst // bytes not copied
75 ret 84 ret
76 .previous 85 .previous
diff --git a/arch/arm64/lib/copy_template.S b/arch/arm64/lib/copy_template.S
new file mode 100644
index 000000000000..410fbdb8163f
--- /dev/null
+++ b/arch/arm64/lib/copy_template.S
@@ -0,0 +1,193 @@
1/*
2 * Copyright (C) 2013 ARM Ltd.
3 * Copyright (C) 2013 Linaro.
4 *
5 * This code is based on glibc cortex strings work originally authored by Linaro
6 * and re-licensed under GPLv2 for the Linux kernel. The original code can
7 * be found @
8 *
9 * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10 * files/head:/src/aarch64/
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program. If not, see <http://www.gnu.org/licenses/>.
23 */
24
25
26/*
27 * Copy a buffer from src to dest (alignment handled by the hardware)
28 *
29 * Parameters:
30 * x0 - dest
31 * x1 - src
32 * x2 - n
33 * Returns:
34 * x0 - dest
35 */
36dstin .req x0
37src .req x1
38count .req x2
39tmp1 .req x3
40tmp1w .req w3
41tmp2 .req x4
42tmp2w .req w4
43dst .req x6
44
45A_l .req x7
46A_h .req x8
47B_l .req x9
48B_h .req x10
49C_l .req x11
50C_h .req x12
51D_l .req x13
52D_h .req x14
53
54 mov dst, dstin
55 cmp count, #16
56 /*When memory length is less than 16, the accessed are not aligned.*/
57 b.lo .Ltiny15
58
59 neg tmp2, src
60 ands tmp2, tmp2, #15/* Bytes to reach alignment. */
61 b.eq .LSrcAligned
62 sub count, count, tmp2
63 /*
64 * Copy the leading memory data from src to dst in an increasing
65 * address order.By this way,the risk of overwritting the source
66 * memory data is eliminated when the distance between src and
67 * dst is less than 16. The memory accesses here are alignment.
68 */
69 tbz tmp2, #0, 1f
70 ldrb1 tmp1w, src, #1
71 strb1 tmp1w, dst, #1
721:
73 tbz tmp2, #1, 2f
74 ldrh1 tmp1w, src, #2
75 strh1 tmp1w, dst, #2
762:
77 tbz tmp2, #2, 3f
78 ldr1 tmp1w, src, #4
79 str1 tmp1w, dst, #4
803:
81 tbz tmp2, #3, .LSrcAligned
82 ldr1 tmp1, src, #8
83 str1 tmp1, dst, #8
84
85.LSrcAligned:
86 cmp count, #64
87 b.ge .Lcpy_over64
88 /*
89 * Deal with small copies quickly by dropping straight into the
90 * exit block.
91 */
92.Ltail63:
93 /*
94 * Copy up to 48 bytes of data. At this point we only need the
95 * bottom 6 bits of count to be accurate.
96 */
97 ands tmp1, count, #0x30
98 b.eq .Ltiny15
99 cmp tmp1w, #0x20
100 b.eq 1f
101 b.lt 2f
102 ldp1 A_l, A_h, src, #16
103 stp1 A_l, A_h, dst, #16
1041:
105 ldp1 A_l, A_h, src, #16
106 stp1 A_l, A_h, dst, #16
1072:
108 ldp1 A_l, A_h, src, #16
109 stp1 A_l, A_h, dst, #16
110.Ltiny15:
111 /*
112 * Prefer to break one ldp/stp into several load/store to access
113 * memory in an increasing address order,rather than to load/store 16
114 * bytes from (src-16) to (dst-16) and to backward the src to aligned
115 * address,which way is used in original cortex memcpy. If keeping
116 * the original memcpy process here, memmove need to satisfy the
117 * precondition that src address is at least 16 bytes bigger than dst
118 * address,otherwise some source data will be overwritten when memove
119 * call memcpy directly. To make memmove simpler and decouple the
120 * memcpy's dependency on memmove, withdrew the original process.
121 */
122 tbz count, #3, 1f
123 ldr1 tmp1, src, #8
124 str1 tmp1, dst, #8
1251:
126 tbz count, #2, 2f
127 ldr1 tmp1w, src, #4
128 str1 tmp1w, dst, #4
1292:
130 tbz count, #1, 3f
131 ldrh1 tmp1w, src, #2
132 strh1 tmp1w, dst, #2
1333:
134 tbz count, #0, .Lexitfunc
135 ldrb1 tmp1w, src, #1
136 strb1 tmp1w, dst, #1
137
138 b .Lexitfunc
139
140.Lcpy_over64:
141 subs count, count, #128
142 b.ge .Lcpy_body_large
143 /*
144 * Less than 128 bytes to copy, so handle 64 here and then jump
145 * to the tail.
146 */
147 ldp1 A_l, A_h, src, #16
148 stp1 A_l, A_h, dst, #16
149 ldp1 B_l, B_h, src, #16
150 ldp1 C_l, C_h, src, #16
151 stp1 B_l, B_h, dst, #16
152 stp1 C_l, C_h, dst, #16
153 ldp1 D_l, D_h, src, #16
154 stp1 D_l, D_h, dst, #16
155
156 tst count, #0x3f
157 b.ne .Ltail63
158 b .Lexitfunc
159
160 /*
161 * Critical loop. Start at a new cache line boundary. Assuming
162 * 64 bytes per line this ensures the entire loop is in one line.
163 */
164 .p2align L1_CACHE_SHIFT
165.Lcpy_body_large:
166 /* pre-get 64 bytes data. */
167 ldp1 A_l, A_h, src, #16
168 ldp1 B_l, B_h, src, #16
169 ldp1 C_l, C_h, src, #16
170 ldp1 D_l, D_h, src, #16
1711:
172 /*
173 * interlace the load of next 64 bytes data block with store of the last
174 * loaded 64 bytes data.
175 */
176 stp1 A_l, A_h, dst, #16
177 ldp1 A_l, A_h, src, #16
178 stp1 B_l, B_h, dst, #16
179 ldp1 B_l, B_h, src, #16
180 stp1 C_l, C_h, dst, #16
181 ldp1 C_l, C_h, src, #16
182 stp1 D_l, D_h, dst, #16
183 ldp1 D_l, D_h, src, #16
184 subs count, count, #64
185 b.ge 1b
186 stp1 A_l, A_h, dst, #16
187 stp1 B_l, B_h, dst, #16
188 stp1 C_l, C_h, dst, #16
189 stp1 D_l, D_h, dst, #16
190
191 tst count, #0x3f
192 b.ne .Ltail63
193.Lexitfunc:
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index a257b47e2dc4..7512bbbc07ac 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -18,6 +18,7 @@
18 18
19#include <asm/alternative.h> 19#include <asm/alternative.h>
20#include <asm/assembler.h> 20#include <asm/assembler.h>
21#include <asm/cache.h>
21#include <asm/cpufeature.h> 22#include <asm/cpufeature.h>
22#include <asm/sysreg.h> 23#include <asm/sysreg.h>
23 24
@@ -31,44 +32,52 @@
31 * Returns: 32 * Returns:
32 * x0 - bytes not copied 33 * x0 - bytes not copied
33 */ 34 */
35 .macro ldrb1 ptr, regB, val
36 ldrb \ptr, [\regB], \val
37 .endm
38
39 .macro strb1 ptr, regB, val
40 USER(9998f, strb \ptr, [\regB], \val)
41 .endm
42
43 .macro ldrh1 ptr, regB, val
44 ldrh \ptr, [\regB], \val
45 .endm
46
47 .macro strh1 ptr, regB, val
48 USER(9998f, strh \ptr, [\regB], \val)
49 .endm
50
51 .macro ldr1 ptr, regB, val
52 ldr \ptr, [\regB], \val
53 .endm
54
55 .macro str1 ptr, regB, val
56 USER(9998f, str \ptr, [\regB], \val)
57 .endm
58
59 .macro ldp1 ptr, regB, regC, val
60 ldp \ptr, \regB, [\regC], \val
61 .endm
62
63 .macro stp1 ptr, regB, regC, val
64 USER(9998f, stp \ptr, \regB, [\regC], \val)
65 .endm
66
67end .req x5
34ENTRY(__copy_to_user) 68ENTRY(__copy_to_user)
35ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ 69ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
36 CONFIG_ARM64_PAN) 70 CONFIG_ARM64_PAN)
37 add x5, x0, x2 // upper user buffer boundary 71 add end, x0, x2
38 subs x2, x2, #16 72#include "copy_template.S"
39 b.mi 1f
400:
41 ldp x3, x4, [x1], #16
42 subs x2, x2, #16
43USER(9f, stp x3, x4, [x0], #16)
44 b.pl 0b
451: adds x2, x2, #8
46 b.mi 2f
47 ldr x3, [x1], #8
48 sub x2, x2, #8
49USER(9f, str x3, [x0], #8 )
502: adds x2, x2, #4
51 b.mi 3f
52 ldr w3, [x1], #4
53 sub x2, x2, #4
54USER(9f, str w3, [x0], #4 )
553: adds x2, x2, #2
56 b.mi 4f
57 ldrh w3, [x1], #2
58 sub x2, x2, #2
59USER(9f, strh w3, [x0], #2 )
604: adds x2, x2, #1
61 b.mi 5f
62 ldrb w3, [x1]
63USER(9f, strb w3, [x0] )
645: mov x0, #0
65ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ 73ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
66 CONFIG_ARM64_PAN) 74 CONFIG_ARM64_PAN)
75 mov x0, #0
67 ret 76 ret
68ENDPROC(__copy_to_user) 77ENDPROC(__copy_to_user)
69 78
70 .section .fixup,"ax" 79 .section .fixup,"ax"
71 .align 2 80 .align 2
729: sub x0, x5, x0 // bytes not copied 819998: sub x0, end, dst // bytes not copied
73 ret 82 ret
74 .previous 83 .previous
diff --git a/arch/arm64/lib/memchr.S b/arch/arm64/lib/memchr.S
index 8636b7549163..4444c1d25f4b 100644
--- a/arch/arm64/lib/memchr.S
+++ b/arch/arm64/lib/memchr.S
@@ -41,4 +41,4 @@ ENTRY(memchr)
41 ret 41 ret
422: mov x0, #0 422: mov x0, #0
43 ret 43 ret
44ENDPROC(memchr) 44ENDPIPROC(memchr)
diff --git a/arch/arm64/lib/memcmp.S b/arch/arm64/lib/memcmp.S
index 6ea0776ba6de..ffbdec00327d 100644
--- a/arch/arm64/lib/memcmp.S
+++ b/arch/arm64/lib/memcmp.S
@@ -255,4 +255,4 @@ CPU_LE( rev data2, data2 )
255.Lret0: 255.Lret0:
256 mov result, #0 256 mov result, #0
257 ret 257 ret
258ENDPROC(memcmp) 258ENDPIPROC(memcmp)
diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S
index 8a9a96d3ddae..67613937711f 100644
--- a/arch/arm64/lib/memcpy.S
+++ b/arch/arm64/lib/memcpy.S
@@ -36,166 +36,42 @@
36 * Returns: 36 * Returns:
37 * x0 - dest 37 * x0 - dest
38 */ 38 */
39dstin .req x0 39 .macro ldrb1 ptr, regB, val
40src .req x1 40 ldrb \ptr, [\regB], \val
41count .req x2 41 .endm
42tmp1 .req x3
43tmp1w .req w3
44tmp2 .req x4
45tmp2w .req w4
46tmp3 .req x5
47tmp3w .req w5
48dst .req x6
49 42
50A_l .req x7 43 .macro strb1 ptr, regB, val
51A_h .req x8 44 strb \ptr, [\regB], \val
52B_l .req x9 45 .endm
53B_h .req x10
54C_l .req x11
55C_h .req x12
56D_l .req x13
57D_h .req x14
58 46
59ENTRY(memcpy) 47 .macro ldrh1 ptr, regB, val
60 mov dst, dstin 48 ldrh \ptr, [\regB], \val
61 cmp count, #16 49 .endm
62 /*When memory length is less than 16, the accessed are not aligned.*/
63 b.lo .Ltiny15
64 50
65 neg tmp2, src 51 .macro strh1 ptr, regB, val
66 ands tmp2, tmp2, #15/* Bytes to reach alignment. */ 52 strh \ptr, [\regB], \val
67 b.eq .LSrcAligned 53 .endm
68 sub count, count, tmp2
69 /*
70 * Copy the leading memory data from src to dst in an increasing
71 * address order.By this way,the risk of overwritting the source
72 * memory data is eliminated when the distance between src and
73 * dst is less than 16. The memory accesses here are alignment.
74 */
75 tbz tmp2, #0, 1f
76 ldrb tmp1w, [src], #1
77 strb tmp1w, [dst], #1
781:
79 tbz tmp2, #1, 2f
80 ldrh tmp1w, [src], #2
81 strh tmp1w, [dst], #2
822:
83 tbz tmp2, #2, 3f
84 ldr tmp1w, [src], #4
85 str tmp1w, [dst], #4
863:
87 tbz tmp2, #3, .LSrcAligned
88 ldr tmp1, [src],#8
89 str tmp1, [dst],#8
90 54
91.LSrcAligned: 55 .macro ldr1 ptr, regB, val
92 cmp count, #64 56 ldr \ptr, [\regB], \val
93 b.ge .Lcpy_over64 57 .endm
94 /*
95 * Deal with small copies quickly by dropping straight into the
96 * exit block.
97 */
98.Ltail63:
99 /*
100 * Copy up to 48 bytes of data. At this point we only need the
101 * bottom 6 bits of count to be accurate.
102 */
103 ands tmp1, count, #0x30
104 b.eq .Ltiny15
105 cmp tmp1w, #0x20
106 b.eq 1f
107 b.lt 2f
108 ldp A_l, A_h, [src], #16
109 stp A_l, A_h, [dst], #16
1101:
111 ldp A_l, A_h, [src], #16
112 stp A_l, A_h, [dst], #16
1132:
114 ldp A_l, A_h, [src], #16
115 stp A_l, A_h, [dst], #16
116.Ltiny15:
117 /*
118 * Prefer to break one ldp/stp into several load/store to access
119 * memory in an increasing address order,rather than to load/store 16
120 * bytes from (src-16) to (dst-16) and to backward the src to aligned
121 * address,which way is used in original cortex memcpy. If keeping
122 * the original memcpy process here, memmove need to satisfy the
123 * precondition that src address is at least 16 bytes bigger than dst
124 * address,otherwise some source data will be overwritten when memove
125 * call memcpy directly. To make memmove simpler and decouple the
126 * memcpy's dependency on memmove, withdrew the original process.
127 */
128 tbz count, #3, 1f
129 ldr tmp1, [src], #8
130 str tmp1, [dst], #8
1311:
132 tbz count, #2, 2f
133 ldr tmp1w, [src], #4
134 str tmp1w, [dst], #4
1352:
136 tbz count, #1, 3f
137 ldrh tmp1w, [src], #2
138 strh tmp1w, [dst], #2
1393:
140 tbz count, #0, .Lexitfunc
141 ldrb tmp1w, [src]
142 strb tmp1w, [dst]
143 58
144.Lexitfunc: 59 .macro str1 ptr, regB, val
145 ret 60 str \ptr, [\regB], \val
61 .endm
146 62
147.Lcpy_over64: 63 .macro ldp1 ptr, regB, regC, val
148 subs count, count, #128 64 ldp \ptr, \regB, [\regC], \val
149 b.ge .Lcpy_body_large 65 .endm
150 /*
151 * Less than 128 bytes to copy, so handle 64 here and then jump
152 * to the tail.
153 */
154 ldp A_l, A_h, [src],#16
155 stp A_l, A_h, [dst],#16
156 ldp B_l, B_h, [src],#16
157 ldp C_l, C_h, [src],#16
158 stp B_l, B_h, [dst],#16
159 stp C_l, C_h, [dst],#16
160 ldp D_l, D_h, [src],#16
161 stp D_l, D_h, [dst],#16
162 66
163 tst count, #0x3f 67 .macro stp1 ptr, regB, regC, val
164 b.ne .Ltail63 68 stp \ptr, \regB, [\regC], \val
165 ret 69 .endm
166 70
167 /* 71 .weak memcpy
168 * Critical loop. Start at a new cache line boundary. Assuming 72ENTRY(__memcpy)
169 * 64 bytes per line this ensures the entire loop is in one line. 73ENTRY(memcpy)
170 */ 74#include "copy_template.S"
171 .p2align L1_CACHE_SHIFT
172.Lcpy_body_large:
173 /* pre-get 64 bytes data. */
174 ldp A_l, A_h, [src],#16
175 ldp B_l, B_h, [src],#16
176 ldp C_l, C_h, [src],#16
177 ldp D_l, D_h, [src],#16
1781:
179 /*
180 * interlace the load of next 64 bytes data block with store of the last
181 * loaded 64 bytes data.
182 */
183 stp A_l, A_h, [dst],#16
184 ldp A_l, A_h, [src],#16
185 stp B_l, B_h, [dst],#16
186 ldp B_l, B_h, [src],#16
187 stp C_l, C_h, [dst],#16
188 ldp C_l, C_h, [src],#16
189 stp D_l, D_h, [dst],#16
190 ldp D_l, D_h, [src],#16
191 subs count, count, #64
192 b.ge 1b
193 stp A_l, A_h, [dst],#16
194 stp B_l, B_h, [dst],#16
195 stp C_l, C_h, [dst],#16
196 stp D_l, D_h, [dst],#16
197
198 tst count, #0x3f
199 b.ne .Ltail63
200 ret 75 ret
201ENDPROC(memcpy) 76ENDPIPROC(memcpy)
77ENDPROC(__memcpy)
diff --git a/arch/arm64/lib/memmove.S b/arch/arm64/lib/memmove.S
index 57b19ea2dad4..a5a4459013b1 100644
--- a/arch/arm64/lib/memmove.S
+++ b/arch/arm64/lib/memmove.S
@@ -57,12 +57,14 @@ C_h .req x12
57D_l .req x13 57D_l .req x13
58D_h .req x14 58D_h .req x14
59 59
60 .weak memmove
61ENTRY(__memmove)
60ENTRY(memmove) 62ENTRY(memmove)
61 cmp dstin, src 63 cmp dstin, src
62 b.lo memcpy 64 b.lo __memcpy
63 add tmp1, src, count 65 add tmp1, src, count
64 cmp dstin, tmp1 66 cmp dstin, tmp1
65 b.hs memcpy /* No overlap. */ 67 b.hs __memcpy /* No overlap. */
66 68
67 add dst, dstin, count 69 add dst, dstin, count
68 add src, src, count 70 add src, src, count
@@ -194,4 +196,5 @@ ENTRY(memmove)
194 tst count, #0x3f 196 tst count, #0x3f
195 b.ne .Ltail63 197 b.ne .Ltail63
196 ret 198 ret
197ENDPROC(memmove) 199ENDPIPROC(memmove)
200ENDPROC(__memmove)
diff --git a/arch/arm64/lib/memset.S b/arch/arm64/lib/memset.S
index 7c72dfd36b63..f2670a9f218c 100644
--- a/arch/arm64/lib/memset.S
+++ b/arch/arm64/lib/memset.S
@@ -54,6 +54,8 @@ dst .req x8
54tmp3w .req w9 54tmp3w .req w9
55tmp3 .req x9 55tmp3 .req x9
56 56
57 .weak memset
58ENTRY(__memset)
57ENTRY(memset) 59ENTRY(memset)
58 mov dst, dstin /* Preserve return value. */ 60 mov dst, dstin /* Preserve return value. */
59 and A_lw, val, #255 61 and A_lw, val, #255
@@ -213,4 +215,5 @@ ENTRY(memset)
213 ands count, count, zva_bits_x 215 ands count, count, zva_bits_x
214 b.ne .Ltail_maybe_long 216 b.ne .Ltail_maybe_long
215 ret 217 ret
216ENDPROC(memset) 218ENDPIPROC(memset)
219ENDPROC(__memset)
diff --git a/arch/arm64/lib/strcmp.S b/arch/arm64/lib/strcmp.S
index 42f828b06c59..471fe61760ef 100644
--- a/arch/arm64/lib/strcmp.S
+++ b/arch/arm64/lib/strcmp.S
@@ -231,4 +231,4 @@ CPU_BE( orr syndrome, diff, has_nul )
231 lsr data1, data1, #56 231 lsr data1, data1, #56
232 sub result, data1, data2, lsr #56 232 sub result, data1, data2, lsr #56
233 ret 233 ret
234ENDPROC(strcmp) 234ENDPIPROC(strcmp)
diff --git a/arch/arm64/lib/strlen.S b/arch/arm64/lib/strlen.S
index 987b68b9ce44..55ccc8e24c08 100644
--- a/arch/arm64/lib/strlen.S
+++ b/arch/arm64/lib/strlen.S
@@ -123,4 +123,4 @@ CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
123 csinv data1, data1, xzr, le 123 csinv data1, data1, xzr, le
124 csel data2, data2, data2a, le 124 csel data2, data2, data2a, le
125 b .Lrealigned 125 b .Lrealigned
126ENDPROC(strlen) 126ENDPIPROC(strlen)
diff --git a/arch/arm64/lib/strncmp.S b/arch/arm64/lib/strncmp.S
index 0224cf5a5533..e267044761c6 100644
--- a/arch/arm64/lib/strncmp.S
+++ b/arch/arm64/lib/strncmp.S
@@ -307,4 +307,4 @@ CPU_BE( orr syndrome, diff, has_nul )
307.Lret0: 307.Lret0:
308 mov result, #0 308 mov result, #0
309 ret 309 ret
310ENDPROC(strncmp) 310ENDPIPROC(strncmp)
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index 773d37a14039..57f57fde5722 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -4,3 +4,6 @@ obj-y := dma-mapping.o extable.o fault.o init.o \
4 context.o proc.o pageattr.o 4 context.o proc.o pageattr.o
5obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o 5obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
6obj-$(CONFIG_ARM64_PTDUMP) += dump.o 6obj-$(CONFIG_ARM64_PTDUMP) += dump.o
7
8obj-$(CONFIG_KASAN) += kasan_init.o
9KASAN_SANITIZE_kasan_init.o := n
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index eb48d5df4a0f..cfa44a6adc0a 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -98,7 +98,7 @@ ENTRY(__flush_dcache_area)
98 b.lo 1b 98 b.lo 1b
99 dsb sy 99 dsb sy
100 ret 100 ret
101ENDPROC(__flush_dcache_area) 101ENDPIPROC(__flush_dcache_area)
102 102
103/* 103/*
104 * __inval_cache_range(start, end) 104 * __inval_cache_range(start, end)
@@ -131,7 +131,7 @@ __dma_inv_range:
131 b.lo 2b 131 b.lo 2b
132 dsb sy 132 dsb sy
133 ret 133 ret
134ENDPROC(__inval_cache_range) 134ENDPIPROC(__inval_cache_range)
135ENDPROC(__dma_inv_range) 135ENDPROC(__dma_inv_range)
136 136
137/* 137/*
@@ -171,7 +171,7 @@ ENTRY(__dma_flush_range)
171 b.lo 1b 171 b.lo 1b
172 dsb sy 172 dsb sy
173 ret 173 ret
174ENDPROC(__dma_flush_range) 174ENDPIPROC(__dma_flush_range)
175 175
176/* 176/*
177 * __dma_map_area(start, size, dir) 177 * __dma_map_area(start, size, dir)
@@ -184,7 +184,7 @@ ENTRY(__dma_map_area)
184 cmp w2, #DMA_FROM_DEVICE 184 cmp w2, #DMA_FROM_DEVICE
185 b.eq __dma_inv_range 185 b.eq __dma_inv_range
186 b __dma_clean_range 186 b __dma_clean_range
187ENDPROC(__dma_map_area) 187ENDPIPROC(__dma_map_area)
188 188
189/* 189/*
190 * __dma_unmap_area(start, size, dir) 190 * __dma_unmap_area(start, size, dir)
@@ -197,4 +197,4 @@ ENTRY(__dma_unmap_area)
197 cmp w2, #DMA_TO_DEVICE 197 cmp w2, #DMA_TO_DEVICE
198 b.ne __dma_inv_range 198 b.ne __dma_inv_range
199 ret 199 ret
200ENDPROC(__dma_unmap_area) 200ENDPIPROC(__dma_unmap_area)
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index d70ff14dbdbd..f636a2639f03 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -17,135 +17,185 @@
17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */ 18 */
19 19
20#include <linux/init.h> 20#include <linux/bitops.h>
21#include <linux/sched.h> 21#include <linux/sched.h>
22#include <linux/slab.h>
22#include <linux/mm.h> 23#include <linux/mm.h>
23#include <linux/smp.h>
24#include <linux/percpu.h>
25 24
25#include <asm/cpufeature.h>
26#include <asm/mmu_context.h> 26#include <asm/mmu_context.h>
27#include <asm/tlbflush.h> 27#include <asm/tlbflush.h>
28#include <asm/cachetype.h>
29 28
30#define asid_bits(reg) \ 29static u32 asid_bits;
31 (((read_cpuid(ID_AA64MMFR0_EL1) & 0xf0) >> 2) + 8) 30static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
32 31
33#define ASID_FIRST_VERSION (1 << MAX_ASID_BITS) 32static atomic64_t asid_generation;
33static unsigned long *asid_map;
34 34
35static DEFINE_RAW_SPINLOCK(cpu_asid_lock); 35static DEFINE_PER_CPU(atomic64_t, active_asids);
36unsigned int cpu_last_asid = ASID_FIRST_VERSION; 36static DEFINE_PER_CPU(u64, reserved_asids);
37static cpumask_t tlb_flush_pending;
37 38
38/* 39#define ASID_MASK (~GENMASK(asid_bits - 1, 0))
39 * We fork()ed a process, and we need a new context for the child to run in. 40#define ASID_FIRST_VERSION (1UL << asid_bits)
40 */ 41#define NUM_USER_ASIDS ASID_FIRST_VERSION
41void __init_new_context(struct task_struct *tsk, struct mm_struct *mm)
42{
43 mm->context.id = 0;
44 raw_spin_lock_init(&mm->context.id_lock);
45}
46 42
47static void flush_context(void) 43static void flush_context(unsigned int cpu)
48{ 44{
49 /* set the reserved TTBR0 before flushing the TLB */ 45 int i;
50 cpu_set_reserved_ttbr0(); 46 u64 asid;
51 flush_tlb_all();
52 if (icache_is_aivivt())
53 __flush_icache_all();
54}
55 47
56static void set_mm_context(struct mm_struct *mm, unsigned int asid) 48 /* Update the list of reserved ASIDs and the ASID bitmap. */
57{ 49 bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
58 unsigned long flags;
59 50
60 /* 51 /*
61 * Locking needed for multi-threaded applications where the same 52 * Ensure the generation bump is observed before we xchg the
62 * mm->context.id could be set from different CPUs during the 53 * active_asids.
63 * broadcast. This function is also called via IPI so the
64 * mm->context.id_lock has to be IRQ-safe.
65 */ 54 */
66 raw_spin_lock_irqsave(&mm->context.id_lock, flags); 55 smp_wmb();
67 if (likely((mm->context.id ^ cpu_last_asid) >> MAX_ASID_BITS)) { 56
57 for_each_possible_cpu(i) {
58 asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0);
68 /* 59 /*
69 * Old version of ASID found. Set the new one and reset 60 * If this CPU has already been through a
70 * mm_cpumask(mm). 61 * rollover, but hasn't run another task in
62 * the meantime, we must preserve its reserved
63 * ASID, as this is the only trace we have of
64 * the process it is still running.
71 */ 65 */
72 mm->context.id = asid; 66 if (asid == 0)
73 cpumask_clear(mm_cpumask(mm)); 67 asid = per_cpu(reserved_asids, i);
68 __set_bit(asid & ~ASID_MASK, asid_map);
69 per_cpu(reserved_asids, i) = asid;
74 } 70 }
75 raw_spin_unlock_irqrestore(&mm->context.id_lock, flags);
76 71
77 /* 72 /* Queue a TLB invalidate and flush the I-cache if necessary. */
78 * Set the mm_cpumask(mm) bit for the current CPU. 73 cpumask_setall(&tlb_flush_pending);
79 */ 74
80 cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); 75 if (icache_is_aivivt())
76 __flush_icache_all();
81} 77}
82 78
83/* 79static int is_reserved_asid(u64 asid)
84 * Reset the ASID on the current CPU. This function call is broadcast from the 80{
85 * CPU handling the ASID rollover and holding cpu_asid_lock. 81 int cpu;
86 */ 82 for_each_possible_cpu(cpu)
87static void reset_context(void *info) 83 if (per_cpu(reserved_asids, cpu) == asid)
84 return 1;
85 return 0;
86}
87
88static u64 new_context(struct mm_struct *mm, unsigned int cpu)
88{ 89{
89 unsigned int asid; 90 static u32 cur_idx = 1;
90 unsigned int cpu = smp_processor_id(); 91 u64 asid = atomic64_read(&mm->context.id);
91 struct mm_struct *mm = current->active_mm; 92 u64 generation = atomic64_read(&asid_generation);
93
94 if (asid != 0) {
95 /*
96 * If our current ASID was active during a rollover, we
97 * can continue to use it and this was just a false alarm.
98 */
99 if (is_reserved_asid(asid))
100 return generation | (asid & ~ASID_MASK);
101
102 /*
103 * We had a valid ASID in a previous life, so try to re-use
104 * it if possible.
105 */
106 asid &= ~ASID_MASK;
107 if (!__test_and_set_bit(asid, asid_map))
108 goto bump_gen;
109 }
92 110
93 /* 111 /*
94 * current->active_mm could be init_mm for the idle thread immediately 112 * Allocate a free ASID. If we can't find one, take a note of the
95 * after secondary CPU boot or hotplug. TTBR0_EL1 is already set to 113 * currently active ASIDs and mark the TLBs as requiring flushes.
96 * the reserved value, so no need to reset any context. 114 * We always count from ASID #1, as we use ASID #0 when setting a
115 * reserved TTBR0 for the init_mm.
97 */ 116 */
98 if (mm == &init_mm) 117 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
99 return; 118 if (asid != NUM_USER_ASIDS)
119 goto set_asid;
100 120
101 smp_rmb(); 121 /* We're out of ASIDs, so increment the global generation count */
102 asid = cpu_last_asid + cpu; 122 generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION,
123 &asid_generation);
124 flush_context(cpu);
103 125
104 flush_context(); 126 /* We have at least 1 ASID per CPU, so this will always succeed */
105 set_mm_context(mm, asid); 127 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
106 128
107 /* set the new ASID */ 129set_asid:
108 cpu_switch_mm(mm->pgd, mm); 130 __set_bit(asid, asid_map);
131 cur_idx = asid;
132
133bump_gen:
134 asid |= generation;
135 return asid;
109} 136}
110 137
111void __new_context(struct mm_struct *mm) 138void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
112{ 139{
113 unsigned int asid; 140 unsigned long flags;
114 unsigned int bits = asid_bits(); 141 u64 asid;
142
143 asid = atomic64_read(&mm->context.id);
115 144
116 raw_spin_lock(&cpu_asid_lock);
117 /* 145 /*
118 * Check the ASID again, in case the change was broadcast from another 146 * The memory ordering here is subtle. We rely on the control
119 * CPU before we acquired the lock. 147 * dependency between the generation read and the update of
148 * active_asids to ensure that we are synchronised with a
149 * parallel rollover (i.e. this pairs with the smp_wmb() in
150 * flush_context).
120 */ 151 */
121 if (!unlikely((mm->context.id ^ cpu_last_asid) >> MAX_ASID_BITS)) { 152 if (!((asid ^ atomic64_read(&asid_generation)) >> asid_bits)
122 cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); 153 && atomic64_xchg_relaxed(&per_cpu(active_asids, cpu), asid))
123 raw_spin_unlock(&cpu_asid_lock); 154 goto switch_mm_fastpath;
124 return; 155
156 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
157 /* Check that our ASID belongs to the current generation. */
158 asid = atomic64_read(&mm->context.id);
159 if ((asid ^ atomic64_read(&asid_generation)) >> asid_bits) {
160 asid = new_context(mm, cpu);
161 atomic64_set(&mm->context.id, asid);
125 } 162 }
126 /*
127 * At this point, it is guaranteed that the current mm (with an old
128 * ASID) isn't active on any other CPU since the ASIDs are changed
129 * simultaneously via IPI.
130 */
131 asid = ++cpu_last_asid;
132 163
133 /* 164 if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending))
134 * If we've used up all our ASIDs, we need to start a new version and 165 local_flush_tlb_all();
135 * flush the TLB. 166
136 */ 167 atomic64_set(&per_cpu(active_asids, cpu), asid);
137 if (unlikely((asid & ((1 << bits) - 1)) == 0)) { 168 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
138 /* increment the ASID version */ 169
139 cpu_last_asid += (1 << MAX_ASID_BITS) - (1 << bits); 170switch_mm_fastpath:
140 if (cpu_last_asid == 0) 171 cpu_switch_mm(mm->pgd, mm);
141 cpu_last_asid = ASID_FIRST_VERSION; 172}
142 asid = cpu_last_asid + smp_processor_id(); 173
143 flush_context(); 174static int asids_init(void)
144 smp_wmb(); 175{
145 smp_call_function(reset_context, NULL, 1); 176 int fld = cpuid_feature_extract_field(read_cpuid(ID_AA64MMFR0_EL1), 4);
146 cpu_last_asid += NR_CPUS - 1; 177
178 switch (fld) {
179 default:
180 pr_warn("Unknown ASID size (%d); assuming 8-bit\n", fld);
181 /* Fallthrough */
182 case 0:
183 asid_bits = 8;
184 break;
185 case 2:
186 asid_bits = 16;
147 } 187 }
148 188
149 set_mm_context(mm, asid); 189 /* If we end up with more CPUs than ASIDs, expect things to crash */
150 raw_spin_unlock(&cpu_asid_lock); 190 WARN_ON(NUM_USER_ASIDS < num_possible_cpus());
191 atomic64_set(&asid_generation, ASID_FIRST_VERSION);
192 asid_map = kzalloc(BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(*asid_map),
193 GFP_KERNEL);
194 if (!asid_map)
195 panic("Failed to allocate bitmap for %lu ASIDs\n",
196 NUM_USER_ASIDS);
197
198 pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
199 return 0;
151} 200}
201early_initcall(asids_init);
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 99224dcebdc5..7963aa4b5d28 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -18,6 +18,7 @@
18 */ 18 */
19 19
20#include <linux/gfp.h> 20#include <linux/gfp.h>
21#include <linux/acpi.h>
21#include <linux/export.h> 22#include <linux/export.h>
22#include <linux/slab.h> 23#include <linux/slab.h>
23#include <linux/genalloc.h> 24#include <linux/genalloc.h>
@@ -28,9 +29,6 @@
28 29
29#include <asm/cacheflush.h> 30#include <asm/cacheflush.h>
30 31
31struct dma_map_ops *dma_ops;
32EXPORT_SYMBOL(dma_ops);
33
34static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot, 32static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
35 bool coherent) 33 bool coherent)
36{ 34{
@@ -100,7 +98,7 @@ static void *__dma_alloc_coherent(struct device *dev, size_t size,
100 if (IS_ENABLED(CONFIG_ZONE_DMA) && 98 if (IS_ENABLED(CONFIG_ZONE_DMA) &&
101 dev->coherent_dma_mask <= DMA_BIT_MASK(32)) 99 dev->coherent_dma_mask <= DMA_BIT_MASK(32))
102 flags |= GFP_DMA; 100 flags |= GFP_DMA;
103 if (dev_get_cma_area(dev) && (flags & __GFP_WAIT)) { 101 if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) {
104 struct page *page; 102 struct page *page;
105 void *addr; 103 void *addr;
106 104
@@ -148,7 +146,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
148 146
149 size = PAGE_ALIGN(size); 147 size = PAGE_ALIGN(size);
150 148
151 if (!coherent && !(flags & __GFP_WAIT)) { 149 if (!coherent && !gfpflags_allow_blocking(flags)) {
152 struct page *page = NULL; 150 struct page *page = NULL;
153 void *addr = __alloc_from_pool(size, &page, flags); 151 void *addr = __alloc_from_pool(size, &page, flags);
154 152
@@ -515,13 +513,7 @@ EXPORT_SYMBOL(dummy_dma_ops);
515 513
516static int __init arm64_dma_init(void) 514static int __init arm64_dma_init(void)
517{ 515{
518 int ret; 516 return atomic_pool_init();
519
520 dma_ops = &swiotlb_dma_ops;
521
522 ret = atomic_pool_init();
523
524 return ret;
525} 517}
526arch_initcall(arm64_dma_init); 518arch_initcall(arm64_dma_init);
527 519
@@ -533,3 +525,467 @@ static int __init dma_debug_do_init(void)
533 return 0; 525 return 0;
534} 526}
535fs_initcall(dma_debug_do_init); 527fs_initcall(dma_debug_do_init);
528
529
530#ifdef CONFIG_IOMMU_DMA
531#include <linux/dma-iommu.h>
532#include <linux/platform_device.h>
533#include <linux/amba/bus.h>
534
535/* Thankfully, all cache ops are by VA so we can ignore phys here */
536static void flush_page(struct device *dev, const void *virt, phys_addr_t phys)
537{
538 __dma_flush_range(virt, virt + PAGE_SIZE);
539}
540
541static void *__iommu_alloc_attrs(struct device *dev, size_t size,
542 dma_addr_t *handle, gfp_t gfp,
543 struct dma_attrs *attrs)
544{
545 bool coherent = is_device_dma_coherent(dev);
546 int ioprot = dma_direction_to_prot(DMA_BIDIRECTIONAL, coherent);
547 size_t iosize = size;
548 void *addr;
549
550 if (WARN(!dev, "cannot create IOMMU mapping for unknown device\n"))
551 return NULL;
552
553 size = PAGE_ALIGN(size);
554
555 /*
556 * Some drivers rely on this, and we probably don't want the
557 * possibility of stale kernel data being read by devices anyway.
558 */
559 gfp |= __GFP_ZERO;
560
561 if (gfpflags_allow_blocking(gfp)) {
562 struct page **pages;
563 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
564
565 pages = iommu_dma_alloc(dev, iosize, gfp, ioprot, handle,
566 flush_page);
567 if (!pages)
568 return NULL;
569
570 addr = dma_common_pages_remap(pages, size, VM_USERMAP, prot,
571 __builtin_return_address(0));
572 if (!addr)
573 iommu_dma_free(dev, pages, iosize, handle);
574 } else {
575 struct page *page;
576 /*
577 * In atomic context we can't remap anything, so we'll only
578 * get the virtually contiguous buffer we need by way of a
579 * physically contiguous allocation.
580 */
581 if (coherent) {
582 page = alloc_pages(gfp, get_order(size));
583 addr = page ? page_address(page) : NULL;
584 } else {
585 addr = __alloc_from_pool(size, &page, gfp);
586 }
587 if (!addr)
588 return NULL;
589
590 *handle = iommu_dma_map_page(dev, page, 0, iosize, ioprot);
591 if (iommu_dma_mapping_error(dev, *handle)) {
592 if (coherent)
593 __free_pages(page, get_order(size));
594 else
595 __free_from_pool(addr, size);
596 addr = NULL;
597 }
598 }
599 return addr;
600}
601
602static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
603 dma_addr_t handle, struct dma_attrs *attrs)
604{
605 size_t iosize = size;
606
607 size = PAGE_ALIGN(size);
608 /*
609 * @cpu_addr will be one of 3 things depending on how it was allocated:
610 * - A remapped array of pages from iommu_dma_alloc(), for all
611 * non-atomic allocations.
612 * - A non-cacheable alias from the atomic pool, for atomic
613 * allocations by non-coherent devices.
614 * - A normal lowmem address, for atomic allocations by
615 * coherent devices.
616 * Hence how dodgy the below logic looks...
617 */
618 if (__in_atomic_pool(cpu_addr, size)) {
619 iommu_dma_unmap_page(dev, handle, iosize, 0, NULL);
620 __free_from_pool(cpu_addr, size);
621 } else if (is_vmalloc_addr(cpu_addr)){
622 struct vm_struct *area = find_vm_area(cpu_addr);
623
624 if (WARN_ON(!area || !area->pages))
625 return;
626 iommu_dma_free(dev, area->pages, iosize, &handle);
627 dma_common_free_remap(cpu_addr, size, VM_USERMAP);
628 } else {
629 iommu_dma_unmap_page(dev, handle, iosize, 0, NULL);
630 __free_pages(virt_to_page(cpu_addr), get_order(size));
631 }
632}
633
634static int __iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
635 void *cpu_addr, dma_addr_t dma_addr, size_t size,
636 struct dma_attrs *attrs)
637{
638 struct vm_struct *area;
639 int ret;
640
641 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
642 is_device_dma_coherent(dev));
643
644 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
645 return ret;
646
647 area = find_vm_area(cpu_addr);
648 if (WARN_ON(!area || !area->pages))
649 return -ENXIO;
650
651 return iommu_dma_mmap(area->pages, size, vma);
652}
653
654static int __iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
655 void *cpu_addr, dma_addr_t dma_addr,
656 size_t size, struct dma_attrs *attrs)
657{
658 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
659 struct vm_struct *area = find_vm_area(cpu_addr);
660
661 if (WARN_ON(!area || !area->pages))
662 return -ENXIO;
663
664 return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size,
665 GFP_KERNEL);
666}
667
668static void __iommu_sync_single_for_cpu(struct device *dev,
669 dma_addr_t dev_addr, size_t size,
670 enum dma_data_direction dir)
671{
672 phys_addr_t phys;
673
674 if (is_device_dma_coherent(dev))
675 return;
676
677 phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr);
678 __dma_unmap_area(phys_to_virt(phys), size, dir);
679}
680
681static void __iommu_sync_single_for_device(struct device *dev,
682 dma_addr_t dev_addr, size_t size,
683 enum dma_data_direction dir)
684{
685 phys_addr_t phys;
686
687 if (is_device_dma_coherent(dev))
688 return;
689
690 phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr);
691 __dma_map_area(phys_to_virt(phys), size, dir);
692}
693
694static dma_addr_t __iommu_map_page(struct device *dev, struct page *page,
695 unsigned long offset, size_t size,
696 enum dma_data_direction dir,
697 struct dma_attrs *attrs)
698{
699 bool coherent = is_device_dma_coherent(dev);
700 int prot = dma_direction_to_prot(dir, coherent);
701 dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);
702
703 if (!iommu_dma_mapping_error(dev, dev_addr) &&
704 !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
705 __iommu_sync_single_for_device(dev, dev_addr, size, dir);
706
707 return dev_addr;
708}
709
710static void __iommu_unmap_page(struct device *dev, dma_addr_t dev_addr,
711 size_t size, enum dma_data_direction dir,
712 struct dma_attrs *attrs)
713{
714 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
715 __iommu_sync_single_for_cpu(dev, dev_addr, size, dir);
716
717 iommu_dma_unmap_page(dev, dev_addr, size, dir, attrs);
718}
719
720static void __iommu_sync_sg_for_cpu(struct device *dev,
721 struct scatterlist *sgl, int nelems,
722 enum dma_data_direction dir)
723{
724 struct scatterlist *sg;
725 int i;
726
727 if (is_device_dma_coherent(dev))
728 return;
729
730 for_each_sg(sgl, sg, nelems, i)
731 __dma_unmap_area(sg_virt(sg), sg->length, dir);
732}
733
734static void __iommu_sync_sg_for_device(struct device *dev,
735 struct scatterlist *sgl, int nelems,
736 enum dma_data_direction dir)
737{
738 struct scatterlist *sg;
739 int i;
740
741 if (is_device_dma_coherent(dev))
742 return;
743
744 for_each_sg(sgl, sg, nelems, i)
745 __dma_map_area(sg_virt(sg), sg->length, dir);
746}
747
748static int __iommu_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
749 int nelems, enum dma_data_direction dir,
750 struct dma_attrs *attrs)
751{
752 bool coherent = is_device_dma_coherent(dev);
753
754 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
755 __iommu_sync_sg_for_device(dev, sgl, nelems, dir);
756
757 return iommu_dma_map_sg(dev, sgl, nelems,
758 dma_direction_to_prot(dir, coherent));
759}
760
761static void __iommu_unmap_sg_attrs(struct device *dev,
762 struct scatterlist *sgl, int nelems,
763 enum dma_data_direction dir,
764 struct dma_attrs *attrs)
765{
766 if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
767 __iommu_sync_sg_for_cpu(dev, sgl, nelems, dir);
768
769 iommu_dma_unmap_sg(dev, sgl, nelems, dir, attrs);
770}
771
772static struct dma_map_ops iommu_dma_ops = {
773 .alloc = __iommu_alloc_attrs,
774 .free = __iommu_free_attrs,
775 .mmap = __iommu_mmap_attrs,
776 .get_sgtable = __iommu_get_sgtable,
777 .map_page = __iommu_map_page,
778 .unmap_page = __iommu_unmap_page,
779 .map_sg = __iommu_map_sg_attrs,
780 .unmap_sg = __iommu_unmap_sg_attrs,
781 .sync_single_for_cpu = __iommu_sync_single_for_cpu,
782 .sync_single_for_device = __iommu_sync_single_for_device,
783 .sync_sg_for_cpu = __iommu_sync_sg_for_cpu,
784 .sync_sg_for_device = __iommu_sync_sg_for_device,
785 .dma_supported = iommu_dma_supported,
786 .mapping_error = iommu_dma_mapping_error,
787};
788
789/*
790 * TODO: Right now __iommu_setup_dma_ops() gets called too early to do
791 * everything it needs to - the device is only partially created and the
792 * IOMMU driver hasn't seen it yet, so it can't have a group. Thus we
793 * need this delayed attachment dance. Once IOMMU probe ordering is sorted
794 * to move the arch_setup_dma_ops() call later, all the notifier bits below
795 * become unnecessary, and will go away.
796 */
797struct iommu_dma_notifier_data {
798 struct list_head list;
799 struct device *dev;
800 const struct iommu_ops *ops;
801 u64 dma_base;
802 u64 size;
803};
804static LIST_HEAD(iommu_dma_masters);
805static DEFINE_MUTEX(iommu_dma_notifier_lock);
806
807/*
808 * Temporarily "borrow" a domain feature flag to to tell if we had to resort
809 * to creating our own domain here, in case we need to clean it up again.
810 */
811#define __IOMMU_DOMAIN_FAKE_DEFAULT (1U << 31)
812
813static bool do_iommu_attach(struct device *dev, const struct iommu_ops *ops,
814 u64 dma_base, u64 size)
815{
816 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
817
818 /*
819 * Best case: The device is either part of a group which was
820 * already attached to a domain in a previous call, or it's
821 * been put in a default DMA domain by the IOMMU core.
822 */
823 if (!domain) {
824 /*
825 * Urgh. The IOMMU core isn't going to do default domains
826 * for non-PCI devices anyway, until it has some means of
827 * abstracting the entirely implementation-specific
828 * sideband data/SoC topology/unicorn dust that may or
829 * may not differentiate upstream masters.
830 * So until then, HORRIBLE HACKS!
831 */
832 domain = ops->domain_alloc(IOMMU_DOMAIN_DMA);
833 if (!domain)
834 goto out_no_domain;
835
836 domain->ops = ops;
837 domain->type = IOMMU_DOMAIN_DMA | __IOMMU_DOMAIN_FAKE_DEFAULT;
838
839 if (iommu_attach_device(domain, dev))
840 goto out_put_domain;
841 }
842
843 if (iommu_dma_init_domain(domain, dma_base, size))
844 goto out_detach;
845
846 dev->archdata.dma_ops = &iommu_dma_ops;
847 return true;
848
849out_detach:
850 iommu_detach_device(domain, dev);
851out_put_domain:
852 if (domain->type & __IOMMU_DOMAIN_FAKE_DEFAULT)
853 iommu_domain_free(domain);
854out_no_domain:
855 pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
856 dev_name(dev));
857 return false;
858}
859
860static void queue_iommu_attach(struct device *dev, const struct iommu_ops *ops,
861 u64 dma_base, u64 size)
862{
863 struct iommu_dma_notifier_data *iommudata;
864
865 iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL);
866 if (!iommudata)
867 return;
868
869 iommudata->dev = dev;
870 iommudata->ops = ops;
871 iommudata->dma_base = dma_base;
872 iommudata->size = size;
873
874 mutex_lock(&iommu_dma_notifier_lock);
875 list_add(&iommudata->list, &iommu_dma_masters);
876 mutex_unlock(&iommu_dma_notifier_lock);
877}
878
879static int __iommu_attach_notifier(struct notifier_block *nb,
880 unsigned long action, void *data)
881{
882 struct iommu_dma_notifier_data *master, *tmp;
883
884 if (action != BUS_NOTIFY_ADD_DEVICE)
885 return 0;
886
887 mutex_lock(&iommu_dma_notifier_lock);
888 list_for_each_entry_safe(master, tmp, &iommu_dma_masters, list) {
889 if (do_iommu_attach(master->dev, master->ops,
890 master->dma_base, master->size)) {
891 list_del(&master->list);
892 kfree(master);
893 }
894 }
895 mutex_unlock(&iommu_dma_notifier_lock);
896 return 0;
897}
898
899static int register_iommu_dma_ops_notifier(struct bus_type *bus)
900{
901 struct notifier_block *nb = kzalloc(sizeof(*nb), GFP_KERNEL);
902 int ret;
903
904 if (!nb)
905 return -ENOMEM;
906 /*
907 * The device must be attached to a domain before the driver probe
908 * routine gets a chance to start allocating DMA buffers. However,
909 * the IOMMU driver also needs a chance to configure the iommu_group
910 * via its add_device callback first, so we need to make the attach
911 * happen between those two points. Since the IOMMU core uses a bus
912 * notifier with default priority for add_device, do the same but
913 * with a lower priority to ensure the appropriate ordering.
914 */
915 nb->notifier_call = __iommu_attach_notifier;
916 nb->priority = -100;
917
918 ret = bus_register_notifier(bus, nb);
919 if (ret) {
920 pr_warn("Failed to register DMA domain notifier; IOMMU DMA ops unavailable on bus '%s'\n",
921 bus->name);
922 kfree(nb);
923 }
924 return ret;
925}
926
927static int __init __iommu_dma_init(void)
928{
929 int ret;
930
931 ret = iommu_dma_init();
932 if (!ret)
933 ret = register_iommu_dma_ops_notifier(&platform_bus_type);
934 if (!ret)
935 ret = register_iommu_dma_ops_notifier(&amba_bustype);
936 return ret;
937}
938arch_initcall(__iommu_dma_init);
939
940static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
941 const struct iommu_ops *ops)
942{
943 struct iommu_group *group;
944
945 if (!ops)
946 return;
947 /*
948 * TODO: As a concession to the future, we're ready to handle being
949 * called both early and late (i.e. after bus_add_device). Once all
950 * the platform bus code is reworked to call us late and the notifier
951 * junk above goes away, move the body of do_iommu_attach here.
952 */
953 group = iommu_group_get(dev);
954 if (group) {
955 do_iommu_attach(dev, ops, dma_base, size);
956 iommu_group_put(group);
957 } else {
958 queue_iommu_attach(dev, ops, dma_base, size);
959 }
960}
961
962void arch_teardown_dma_ops(struct device *dev)
963{
964 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
965
966 if (domain) {
967 iommu_detach_device(domain, dev);
968 if (domain->type & __IOMMU_DOMAIN_FAKE_DEFAULT)
969 iommu_domain_free(domain);
970 }
971
972 dev->archdata.dma_ops = NULL;
973}
974
975#else
976
977static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
978 struct iommu_ops *iommu)
979{ }
980
981#endif /* CONFIG_IOMMU_DMA */
982
983void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
984 struct iommu_ops *iommu, bool coherent)
985{
986 if (!dev->archdata.dma_ops)
987 dev->archdata.dma_ops = &swiotlb_dma_ops;
988
989 dev->archdata.dma_coherent = coherent;
990 __iommu_setup_dma_ops(dev, dma_base, size, iommu);
991}
diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index f3d6221cd5bd..5a22a119a74c 100644
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -67,6 +67,12 @@ static struct addr_marker address_markers[] = {
67 { -1, NULL }, 67 { -1, NULL },
68}; 68};
69 69
70/*
71 * The page dumper groups page table entries of the same type into a single
72 * description. It uses pg_state to track the range information while
73 * iterating over the pte entries. When the continuity is broken it then
74 * dumps out a description of the range.
75 */
70struct pg_state { 76struct pg_state {
71 struct seq_file *seq; 77 struct seq_file *seq;
72 const struct addr_marker *marker; 78 const struct addr_marker *marker;
@@ -114,6 +120,16 @@ static const struct prot_bits pte_bits[] = {
114 .set = "NG", 120 .set = "NG",
115 .clear = " ", 121 .clear = " ",
116 }, { 122 }, {
123 .mask = PTE_CONT,
124 .val = PTE_CONT,
125 .set = "CON",
126 .clear = " ",
127 }, {
128 .mask = PTE_TABLE_BIT,
129 .val = PTE_TABLE_BIT,
130 .set = " ",
131 .clear = "BLK",
132 }, {
117 .mask = PTE_UXN, 133 .mask = PTE_UXN,
118 .val = PTE_UXN, 134 .val = PTE_UXN,
119 .set = "UXN", 135 .set = "UXN",
@@ -198,7 +214,7 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level,
198 unsigned long delta; 214 unsigned long delta;
199 215
200 if (st->current_prot) { 216 if (st->current_prot) {
201 seq_printf(st->seq, "0x%16lx-0x%16lx ", 217 seq_printf(st->seq, "0x%016lx-0x%016lx ",
202 st->start_address, addr); 218 st->start_address, addr);
203 219
204 delta = (addr - st->start_address) >> 10; 220 delta = (addr - st->start_address) >> 10;
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 9fadf6d7039b..19211c4a8911 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -556,7 +556,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
556} 556}
557 557
558#ifdef CONFIG_ARM64_PAN 558#ifdef CONFIG_ARM64_PAN
559void cpu_enable_pan(void) 559void cpu_enable_pan(void *__unused)
560{ 560{
561 config_sctlr_el1(SCTLR_EL1_SPAN, 0); 561 config_sctlr_el1(SCTLR_EL1_SPAN, 0);
562} 562}
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index f5c0680d17d9..17bf39ac83ba 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -86,10 +86,10 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
86 memset(zone_size, 0, sizeof(zone_size)); 86 memset(zone_size, 0, sizeof(zone_size));
87 87
88 /* 4GB maximum for 32-bit only capable devices */ 88 /* 4GB maximum for 32-bit only capable devices */
89 if (IS_ENABLED(CONFIG_ZONE_DMA)) { 89#ifdef CONFIG_ZONE_DMA
90 max_dma = PFN_DOWN(arm64_dma_phys_limit); 90 max_dma = PFN_DOWN(arm64_dma_phys_limit);
91 zone_size[ZONE_DMA] = max_dma - min; 91 zone_size[ZONE_DMA] = max_dma - min;
92 } 92#endif
93 zone_size[ZONE_NORMAL] = max - max_dma; 93 zone_size[ZONE_NORMAL] = max - max_dma;
94 94
95 memcpy(zhole_size, zone_size, sizeof(zhole_size)); 95 memcpy(zhole_size, zone_size, sizeof(zhole_size));
@@ -101,11 +101,12 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
101 if (start >= max) 101 if (start >= max)
102 continue; 102 continue;
103 103
104 if (IS_ENABLED(CONFIG_ZONE_DMA) && start < max_dma) { 104#ifdef CONFIG_ZONE_DMA
105 if (start < max_dma) {
105 unsigned long dma_end = min(end, max_dma); 106 unsigned long dma_end = min(end, max_dma);
106 zhole_size[ZONE_DMA] -= dma_end - start; 107 zhole_size[ZONE_DMA] -= dma_end - start;
107 } 108 }
108 109#endif
109 if (end > max_dma) { 110 if (end > max_dma) {
110 unsigned long normal_end = min(end, max); 111 unsigned long normal_end = min(end, max);
111 unsigned long normal_start = max(start, max_dma); 112 unsigned long normal_start = max(start, max_dma);
@@ -298,6 +299,9 @@ void __init mem_init(void)
298#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K) 299#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
299 300
300 pr_notice("Virtual kernel memory layout:\n" 301 pr_notice("Virtual kernel memory layout:\n"
302#ifdef CONFIG_KASAN
303 " kasan : 0x%16lx - 0x%16lx (%6ld GB)\n"
304#endif
301 " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n" 305 " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n"
302#ifdef CONFIG_SPARSEMEM_VMEMMAP 306#ifdef CONFIG_SPARSEMEM_VMEMMAP
303 " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" 307 " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
@@ -310,6 +314,9 @@ void __init mem_init(void)
310 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n" 314 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
311 " .text : 0x%p" " - 0x%p" " (%6ld KB)\n" 315 " .text : 0x%p" " - 0x%p" " (%6ld KB)\n"
312 " .data : 0x%p" " - 0x%p" " (%6ld KB)\n", 316 " .data : 0x%p" " - 0x%p" " (%6ld KB)\n",
317#ifdef CONFIG_KASAN
318 MLG(KASAN_SHADOW_START, KASAN_SHADOW_END),
319#endif
313 MLG(VMALLOC_START, VMALLOC_END), 320 MLG(VMALLOC_START, VMALLOC_END),
314#ifdef CONFIG_SPARSEMEM_VMEMMAP 321#ifdef CONFIG_SPARSEMEM_VMEMMAP
315 MLG((unsigned long)vmemmap, 322 MLG((unsigned long)vmemmap,
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
new file mode 100644
index 000000000000..cf038c7d9fa9
--- /dev/null
+++ b/arch/arm64/mm/kasan_init.c
@@ -0,0 +1,165 @@
1/*
2 * This file contains kasan initialization code for ARM64.
3 *
4 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
5 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 */
12
13#define pr_fmt(fmt) "kasan: " fmt
14#include <linux/kasan.h>
15#include <linux/kernel.h>
16#include <linux/memblock.h>
17#include <linux/start_kernel.h>
18
19#include <asm/page.h>
20#include <asm/pgalloc.h>
21#include <asm/pgtable.h>
22#include <asm/tlbflush.h>
23
24static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
25
26static void __init kasan_early_pte_populate(pmd_t *pmd, unsigned long addr,
27 unsigned long end)
28{
29 pte_t *pte;
30 unsigned long next;
31
32 if (pmd_none(*pmd))
33 pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
34
35 pte = pte_offset_kernel(pmd, addr);
36 do {
37 next = addr + PAGE_SIZE;
38 set_pte(pte, pfn_pte(virt_to_pfn(kasan_zero_page),
39 PAGE_KERNEL));
40 } while (pte++, addr = next, addr != end && pte_none(*pte));
41}
42
43static void __init kasan_early_pmd_populate(pud_t *pud,
44 unsigned long addr,
45 unsigned long end)
46{
47 pmd_t *pmd;
48 unsigned long next;
49
50 if (pud_none(*pud))
51 pud_populate(&init_mm, pud, kasan_zero_pmd);
52
53 pmd = pmd_offset(pud, addr);
54 do {
55 next = pmd_addr_end(addr, end);
56 kasan_early_pte_populate(pmd, addr, next);
57 } while (pmd++, addr = next, addr != end && pmd_none(*pmd));
58}
59
60static void __init kasan_early_pud_populate(pgd_t *pgd,
61 unsigned long addr,
62 unsigned long end)
63{
64 pud_t *pud;
65 unsigned long next;
66
67 if (pgd_none(*pgd))
68 pgd_populate(&init_mm, pgd, kasan_zero_pud);
69
70 pud = pud_offset(pgd, addr);
71 do {
72 next = pud_addr_end(addr, end);
73 kasan_early_pmd_populate(pud, addr, next);
74 } while (pud++, addr = next, addr != end && pud_none(*pud));
75}
76
77static void __init kasan_map_early_shadow(void)
78{
79 unsigned long addr = KASAN_SHADOW_START;
80 unsigned long end = KASAN_SHADOW_END;
81 unsigned long next;
82 pgd_t *pgd;
83
84 pgd = pgd_offset_k(addr);
85 do {
86 next = pgd_addr_end(addr, end);
87 kasan_early_pud_populate(pgd, addr, next);
88 } while (pgd++, addr = next, addr != end);
89}
90
91asmlinkage void __init kasan_early_init(void)
92{
93 BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61));
94 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE));
95 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
96 kasan_map_early_shadow();
97}
98
99static void __init clear_pgds(unsigned long start,
100 unsigned long end)
101{
102 /*
103 * Remove references to kasan page tables from
104 * swapper_pg_dir. pgd_clear() can't be used
105 * here because it's nop on 2,3-level pagetable setups
106 */
107 for (; start < end; start += PGDIR_SIZE)
108 set_pgd(pgd_offset_k(start), __pgd(0));
109}
110
111static void __init cpu_set_ttbr1(unsigned long ttbr1)
112{
113 asm(
114 " msr ttbr1_el1, %0\n"
115 " isb"
116 :
117 : "r" (ttbr1));
118}
119
120void __init kasan_init(void)
121{
122 struct memblock_region *reg;
123
124 /*
125 * We are going to perform proper setup of shadow memory.
126 * At first we should unmap early shadow (clear_pgds() call bellow).
127 * However, instrumented code couldn't execute without shadow memory.
128 * tmp_pg_dir used to keep early shadow mapped until full shadow
129 * setup will be finished.
130 */
131 memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
132 cpu_set_ttbr1(__pa(tmp_pg_dir));
133 flush_tlb_all();
134
135 clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
136
137 kasan_populate_zero_shadow((void *)KASAN_SHADOW_START,
138 kasan_mem_to_shadow((void *)MODULES_VADDR));
139
140 for_each_memblock(memory, reg) {
141 void *start = (void *)__phys_to_virt(reg->base);
142 void *end = (void *)__phys_to_virt(reg->base + reg->size);
143
144 if (start >= end)
145 break;
146
147 /*
148 * end + 1 here is intentional. We check several shadow bytes in
149 * advance to slightly speed up fastpath. In some rare cases
150 * we could cross boundary of mapped shadow, so we just map
151 * some more here.
152 */
153 vmemmap_populate((unsigned long)kasan_mem_to_shadow(start),
154 (unsigned long)kasan_mem_to_shadow(end) + 1,
155 pfn_to_nid(virt_to_pfn(start)));
156 }
157
158 memset(kasan_zero_page, 0, PAGE_SIZE);
159 cpu_set_ttbr1(__pa(swapper_pg_dir));
160 flush_tlb_all();
161
162 /* At this point kasan is fully initialized. Enable error messages */
163 init_task.kasan_depth = 0;
164 pr_info("KernelAddressSanitizer initialized\n");
165}
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9211b8527f25..abb66f84d4ac 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -32,6 +32,7 @@
32 32
33#include <asm/cputype.h> 33#include <asm/cputype.h>
34#include <asm/fixmap.h> 34#include <asm/fixmap.h>
35#include <asm/kernel-pgtable.h>
35#include <asm/sections.h> 36#include <asm/sections.h>
36#include <asm/setup.h> 37#include <asm/setup.h>
37#include <asm/sizes.h> 38#include <asm/sizes.h>
@@ -80,19 +81,55 @@ static void split_pmd(pmd_t *pmd, pte_t *pte)
80 do { 81 do {
81 /* 82 /*
82 * Need to have the least restrictive permissions available 83 * Need to have the least restrictive permissions available
83 * permissions will be fixed up later 84 * permissions will be fixed up later. Default the new page
85 * range as contiguous ptes.
84 */ 86 */
85 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); 87 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC_CONT));
86 pfn++; 88 pfn++;
87 } while (pte++, i++, i < PTRS_PER_PTE); 89 } while (pte++, i++, i < PTRS_PER_PTE);
88} 90}
89 91
92/*
93 * Given a PTE with the CONT bit set, determine where the CONT range
94 * starts, and clear the entire range of PTE CONT bits.
95 */
96static void clear_cont_pte_range(pte_t *pte, unsigned long addr)
97{
98 int i;
99
100 pte -= CONT_RANGE_OFFSET(addr);
101 for (i = 0; i < CONT_PTES; i++) {
102 set_pte(pte, pte_mknoncont(*pte));
103 pte++;
104 }
105 flush_tlb_all();
106}
107
108/*
109 * Given a range of PTEs set the pfn and provided page protection flags
110 */
111static void __populate_init_pte(pte_t *pte, unsigned long addr,
112 unsigned long end, phys_addr_t phys,
113 pgprot_t prot)
114{
115 unsigned long pfn = __phys_to_pfn(phys);
116
117 do {
118 /* clear all the bits except the pfn, then apply the prot */
119 set_pte(pte, pfn_pte(pfn, prot));
120 pte++;
121 pfn++;
122 addr += PAGE_SIZE;
123 } while (addr != end);
124}
125
90static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 126static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
91 unsigned long end, unsigned long pfn, 127 unsigned long end, phys_addr_t phys,
92 pgprot_t prot, 128 pgprot_t prot,
93 void *(*alloc)(unsigned long size)) 129 void *(*alloc)(unsigned long size))
94{ 130{
95 pte_t *pte; 131 pte_t *pte;
132 unsigned long next;
96 133
97 if (pmd_none(*pmd) || pmd_sect(*pmd)) { 134 if (pmd_none(*pmd) || pmd_sect(*pmd)) {
98 pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); 135 pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
@@ -105,12 +142,30 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
105 142
106 pte = pte_offset_kernel(pmd, addr); 143 pte = pte_offset_kernel(pmd, addr);
107 do { 144 do {
108 set_pte(pte, pfn_pte(pfn, prot)); 145 next = min(end, (addr + CONT_SIZE) & CONT_MASK);
109 pfn++; 146 if (((addr | next | phys) & ~CONT_MASK) == 0) {
110 } while (pte++, addr += PAGE_SIZE, addr != end); 147 /* a block of CONT_PTES */
148 __populate_init_pte(pte, addr, next, phys,
149 __pgprot(pgprot_val(prot) | PTE_CONT));
150 } else {
151 /*
152 * If the range being split is already inside of a
153 * contiguous range but this PTE isn't going to be
154 * contiguous, then we want to unmark the adjacent
155 * ranges, then update the portion of the range we
156 * are interrested in.
157 */
158 clear_cont_pte_range(pte, addr);
159 __populate_init_pte(pte, addr, next, phys, prot);
160 }
161
162 pte += (next - addr) >> PAGE_SHIFT;
163 phys += next - addr;
164 addr = next;
165 } while (addr != end);
111} 166}
112 167
113void split_pud(pud_t *old_pud, pmd_t *pmd) 168static void split_pud(pud_t *old_pud, pmd_t *pmd)
114{ 169{
115 unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; 170 unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
116 pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); 171 pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
@@ -168,8 +223,7 @@ static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
168 } 223 }
169 } 224 }
170 } else { 225 } else {
171 alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 226 alloc_init_pte(pmd, addr, next, phys, prot, alloc);
172 prot, alloc);
173 } 227 }
174 phys += next - addr; 228 phys += next - addr;
175 } while (pmd++, addr = next, addr != end); 229 } while (pmd++, addr = next, addr != end);
@@ -308,8 +362,8 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
308 * for now. This will get more fine grained later once all memory 362 * for now. This will get more fine grained later once all memory
309 * is mapped 363 * is mapped
310 */ 364 */
311 unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE); 365 unsigned long kernel_x_start = round_down(__pa(_stext), SWAPPER_BLOCK_SIZE);
312 unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); 366 unsigned long kernel_x_end = round_up(__pa(__init_end), SWAPPER_BLOCK_SIZE);
313 367
314 if (end < kernel_x_start) { 368 if (end < kernel_x_start) {
315 create_mapping(start, __phys_to_virt(start), 369 create_mapping(start, __phys_to_virt(start),
@@ -353,14 +407,11 @@ static void __init map_mem(void)
353 * memory addressable from the initial direct kernel mapping. 407 * memory addressable from the initial direct kernel mapping.
354 * 408 *
355 * The initial direct kernel mapping, located at swapper_pg_dir, gives 409 * The initial direct kernel mapping, located at swapper_pg_dir, gives
356 * us PUD_SIZE (4K pages) or PMD_SIZE (64K pages) memory starting from 410 * us PUD_SIZE (with SECTION maps) or PMD_SIZE (without SECTION maps,
357 * PHYS_OFFSET (which must be aligned to 2MB as per 411 * memory starting from PHYS_OFFSET (which must be aligned to 2MB as
358 * Documentation/arm64/booting.txt). 412 * per Documentation/arm64/booting.txt).
359 */ 413 */
360 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) 414 limit = PHYS_OFFSET + SWAPPER_INIT_MAP_SIZE;
361 limit = PHYS_OFFSET + PMD_SIZE;
362 else
363 limit = PHYS_OFFSET + PUD_SIZE;
364 memblock_set_current_limit(limit); 415 memblock_set_current_limit(limit);
365 416
366 /* map all the memory banks */ 417 /* map all the memory banks */
@@ -371,21 +422,24 @@ static void __init map_mem(void)
371 if (start >= end) 422 if (start >= end)
372 break; 423 break;
373 424
374#ifndef CONFIG_ARM64_64K_PAGES 425 if (ARM64_SWAPPER_USES_SECTION_MAPS) {
375 /* 426 /*
376 * For the first memory bank align the start address and 427 * For the first memory bank align the start address and
377 * current memblock limit to prevent create_mapping() from 428 * current memblock limit to prevent create_mapping() from
378 * allocating pte page tables from unmapped memory. 429 * allocating pte page tables from unmapped memory. With
379 * When 64K pages are enabled, the pte page table for the 430 * the section maps, if the first block doesn't end on section
380 * first PGDIR_SIZE is already present in swapper_pg_dir. 431 * size boundary, create_mapping() will try to allocate a pte
381 */ 432 * page, which may be returned from an unmapped area.
382 if (start < limit) 433 * When section maps are not used, the pte page table for the
383 start = ALIGN(start, PMD_SIZE); 434 * current limit is already present in swapper_pg_dir.
384 if (end < limit) { 435 */
385 limit = end & PMD_MASK; 436 if (start < limit)
386 memblock_set_current_limit(limit); 437 start = ALIGN(start, SECTION_SIZE);
438 if (end < limit) {
439 limit = end & SECTION_MASK;
440 memblock_set_current_limit(limit);
441 }
387 } 442 }
388#endif
389 __map_memblock(start, end); 443 __map_memblock(start, end);
390 } 444 }
391 445
@@ -393,22 +447,22 @@ static void __init map_mem(void)
393 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); 447 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
394} 448}
395 449
396void __init fixup_executable(void) 450static void __init fixup_executable(void)
397{ 451{
398#ifdef CONFIG_DEBUG_RODATA 452#ifdef CONFIG_DEBUG_RODATA
399 /* now that we are actually fully mapped, make the start/end more fine grained */ 453 /* now that we are actually fully mapped, make the start/end more fine grained */
400 if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) { 454 if (!IS_ALIGNED((unsigned long)_stext, SWAPPER_BLOCK_SIZE)) {
401 unsigned long aligned_start = round_down(__pa(_stext), 455 unsigned long aligned_start = round_down(__pa(_stext),
402 SECTION_SIZE); 456 SWAPPER_BLOCK_SIZE);
403 457
404 create_mapping(aligned_start, __phys_to_virt(aligned_start), 458 create_mapping(aligned_start, __phys_to_virt(aligned_start),
405 __pa(_stext) - aligned_start, 459 __pa(_stext) - aligned_start,
406 PAGE_KERNEL); 460 PAGE_KERNEL);
407 } 461 }
408 462
409 if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) { 463 if (!IS_ALIGNED((unsigned long)__init_end, SWAPPER_BLOCK_SIZE)) {
410 unsigned long aligned_end = round_up(__pa(__init_end), 464 unsigned long aligned_end = round_up(__pa(__init_end),
411 SECTION_SIZE); 465 SWAPPER_BLOCK_SIZE);
412 create_mapping(__pa(__init_end), (unsigned long)__init_end, 466 create_mapping(__pa(__init_end), (unsigned long)__init_end,
413 aligned_end - __pa(__init_end), 467 aligned_end - __pa(__init_end),
414 PAGE_KERNEL); 468 PAGE_KERNEL);
@@ -421,7 +475,7 @@ void mark_rodata_ro(void)
421{ 475{
422 create_mapping_late(__pa(_stext), (unsigned long)_stext, 476 create_mapping_late(__pa(_stext), (unsigned long)_stext,
423 (unsigned long)_etext - (unsigned long)_stext, 477 (unsigned long)_etext - (unsigned long)_stext,
424 PAGE_KERNEL_EXEC | PTE_RDONLY); 478 PAGE_KERNEL_ROX);
425 479
426} 480}
427#endif 481#endif
@@ -456,7 +510,7 @@ void __init paging_init(void)
456 * point to zero page to avoid speculatively fetching new entries. 510 * point to zero page to avoid speculatively fetching new entries.
457 */ 511 */
458 cpu_set_reserved_ttbr0(); 512 cpu_set_reserved_ttbr0();
459 flush_tlb_all(); 513 local_flush_tlb_all();
460 cpu_set_default_tcr_t0sz(); 514 cpu_set_default_tcr_t0sz();
461} 515}
462 516
@@ -498,12 +552,12 @@ int kern_addr_valid(unsigned long addr)
498 return pfn_valid(pte_pfn(*pte)); 552 return pfn_valid(pte_pfn(*pte));
499} 553}
500#ifdef CONFIG_SPARSEMEM_VMEMMAP 554#ifdef CONFIG_SPARSEMEM_VMEMMAP
501#ifdef CONFIG_ARM64_64K_PAGES 555#if !ARM64_SWAPPER_USES_SECTION_MAPS
502int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 556int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
503{ 557{
504 return vmemmap_populate_basepages(start, end, node); 558 return vmemmap_populate_basepages(start, end, node);
505} 559}
506#else /* !CONFIG_ARM64_64K_PAGES */ 560#else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
507int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 561int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
508{ 562{
509 unsigned long addr = start; 563 unsigned long addr = start;
@@ -637,8 +691,8 @@ void __set_fixmap(enum fixed_addresses idx,
637void *__init fixmap_remap_fdt(phys_addr_t dt_phys) 691void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
638{ 692{
639 const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 693 const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
640 pgprot_t prot = PAGE_KERNEL | PTE_RDONLY; 694 pgprot_t prot = PAGE_KERNEL_RO;
641 int granularity, size, offset; 695 int size, offset;
642 void *dt_virt; 696 void *dt_virt;
643 697
644 /* 698 /*
@@ -664,24 +718,15 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
664 */ 718 */
665 BUILD_BUG_ON(dt_virt_base % SZ_2M); 719 BUILD_BUG_ON(dt_virt_base % SZ_2M);
666 720
667 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) { 721 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
668 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PMD_SHIFT != 722 __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
669 __fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT);
670
671 granularity = PAGE_SIZE;
672 } else {
673 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PUD_SHIFT !=
674 __fix_to_virt(FIX_BTMAP_BEGIN) >> PUD_SHIFT);
675
676 granularity = PMD_SIZE;
677 }
678 723
679 offset = dt_phys % granularity; 724 offset = dt_phys % SWAPPER_BLOCK_SIZE;
680 dt_virt = (void *)dt_virt_base + offset; 725 dt_virt = (void *)dt_virt_base + offset;
681 726
682 /* map the first chunk so we can read the size from the header */ 727 /* map the first chunk so we can read the size from the header */
683 create_mapping(round_down(dt_phys, granularity), dt_virt_base, 728 create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
684 granularity, prot); 729 SWAPPER_BLOCK_SIZE, prot);
685 730
686 if (fdt_check_header(dt_virt) != 0) 731 if (fdt_check_header(dt_virt) != 0)
687 return NULL; 732 return NULL;
@@ -690,9 +735,9 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
690 if (size > MAX_FDT_SIZE) 735 if (size > MAX_FDT_SIZE)
691 return NULL; 736 return NULL;
692 737
693 if (offset + size > granularity) 738 if (offset + size > SWAPPER_BLOCK_SIZE)
694 create_mapping(round_down(dt_phys, granularity), dt_virt_base, 739 create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
695 round_up(offset + size, granularity), prot); 740 round_up(offset + size, SWAPPER_BLOCK_SIZE), prot);
696 741
697 memblock_reserve(dt_phys, size); 742 memblock_reserve(dt_phys, size);
698 743
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index e47ed1c5dce1..3571c7309c5e 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -45,7 +45,7 @@ static int change_memory_common(unsigned long addr, int numpages,
45 int ret; 45 int ret;
46 struct page_change_data data; 46 struct page_change_data data;
47 47
48 if (!IS_ALIGNED(addr, PAGE_SIZE)) { 48 if (!PAGE_ALIGNED(addr)) {
49 start &= PAGE_MASK; 49 start &= PAGE_MASK;
50 end = start + size; 50 end = start + size;
51 WARN_ON_ONCE(1); 51 WARN_ON_ONCE(1);
diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 71ca104f97bd..cb3ba1b812e7 100644
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -28,8 +28,6 @@
28 28
29#include "mm.h" 29#include "mm.h"
30 30
31#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
32
33static struct kmem_cache *pgd_cache; 31static struct kmem_cache *pgd_cache;
34 32
35pgd_t *pgd_alloc(struct mm_struct *mm) 33pgd_t *pgd_alloc(struct mm_struct *mm)
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index e4ee7bd8830a..cacecc4ad3e5 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -30,7 +30,9 @@
30 30
31#ifdef CONFIG_ARM64_64K_PAGES 31#ifdef CONFIG_ARM64_64K_PAGES
32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K 32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
33#else 33#elif defined(CONFIG_ARM64_16K_PAGES)
34#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
35#else /* CONFIG_ARM64_4K_PAGES */
34#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K 36#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
35#endif 37#endif
36 38
@@ -130,7 +132,7 @@ ENDPROC(cpu_do_resume)
130 * - pgd_phys - physical address of new TTB 132 * - pgd_phys - physical address of new TTB
131 */ 133 */
132ENTRY(cpu_do_switch_mm) 134ENTRY(cpu_do_switch_mm)
133 mmid w1, x1 // get mm->context.id 135 mmid x1, x1 // get mm->context.id
134 bfi x0, x1, #48, #16 // set the ASID 136 bfi x0, x1, #48, #16 // set the ASID
135 msr ttbr0_el1, x0 // set TTBR0 137 msr ttbr0_el1, x0 // set TTBR0
136 isb 138 isb
@@ -146,8 +148,8 @@ ENDPROC(cpu_do_switch_mm)
146 * value of the SCTLR_EL1 register. 148 * value of the SCTLR_EL1 register.
147 */ 149 */
148ENTRY(__cpu_setup) 150ENTRY(__cpu_setup)
149 tlbi vmalle1is // invalidate I + D TLBs 151 tlbi vmalle1 // Invalidate local TLB
150 dsb ish 152 dsb nsh
151 153
152 mov x0, #3 << 20 154 mov x0, #3 << 20
153 msr cpacr_el1, x0 // Enable FP/ASIMD 155 msr cpacr_el1, x0 // Enable FP/ASIMD
@@ -163,12 +165,14 @@ ENTRY(__cpu_setup)
163 * DEVICE_GRE 010 00001100 165 * DEVICE_GRE 010 00001100
164 * NORMAL_NC 011 01000100 166 * NORMAL_NC 011 01000100
165 * NORMAL 100 11111111 167 * NORMAL 100 11111111
168 * NORMAL_WT 101 10111011
166 */ 169 */
167 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \ 170 ldr x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
168 MAIR(0x04, MT_DEVICE_nGnRE) | \ 171 MAIR(0x04, MT_DEVICE_nGnRE) | \
169 MAIR(0x0c, MT_DEVICE_GRE) | \ 172 MAIR(0x0c, MT_DEVICE_GRE) | \
170 MAIR(0x44, MT_NORMAL_NC) | \ 173 MAIR(0x44, MT_NORMAL_NC) | \
171 MAIR(0xff, MT_NORMAL) 174 MAIR(0xff, MT_NORMAL) | \
175 MAIR(0xbb, MT_NORMAL_WT)
172 msr mair_el1, x5 176 msr mair_el1, x5
173 /* 177 /*
174 * Prepare SCTLR 178 * Prepare SCTLR
diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h
index 98a26ce82d26..aee5637ea436 100644
--- a/arch/arm64/net/bpf_jit.h
+++ b/arch/arm64/net/bpf_jit.h
@@ -1,7 +1,7 @@
1/* 1/*
2 * BPF JIT compiler for ARM64 2 * BPF JIT compiler for ARM64
3 * 3 *
4 * Copyright (C) 2014 Zi Shen Lim <zlim.lnx@gmail.com> 4 * Copyright (C) 2014-2015 Zi Shen Lim <zlim.lnx@gmail.com>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -35,6 +35,7 @@
35 aarch64_insn_gen_comp_branch_imm(0, offset, Rt, A64_VARIANT(sf), \ 35 aarch64_insn_gen_comp_branch_imm(0, offset, Rt, A64_VARIANT(sf), \
36 AARCH64_INSN_BRANCH_COMP_##type) 36 AARCH64_INSN_BRANCH_COMP_##type)
37#define A64_CBZ(sf, Rt, imm19) A64_COMP_BRANCH(sf, Rt, (imm19) << 2, ZERO) 37#define A64_CBZ(sf, Rt, imm19) A64_COMP_BRANCH(sf, Rt, (imm19) << 2, ZERO)
38#define A64_CBNZ(sf, Rt, imm19) A64_COMP_BRANCH(sf, Rt, (imm19) << 2, NONZERO)
38 39
39/* Conditional branch (immediate) */ 40/* Conditional branch (immediate) */
40#define A64_COND_BRANCH(cond, offset) \ 41#define A64_COND_BRANCH(cond, offset) \
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index c047598b09e0..d6a53ef2350b 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * BPF JIT compiler for ARM64 2 * BPF JIT compiler for ARM64
3 * 3 *
4 * Copyright (C) 2014 Zi Shen Lim <zlim.lnx@gmail.com> 4 * Copyright (C) 2014-2015 Zi Shen Lim <zlim.lnx@gmail.com>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -50,7 +50,7 @@ static const int bpf2a64[] = {
50 [BPF_REG_8] = A64_R(21), 50 [BPF_REG_8] = A64_R(21),
51 [BPF_REG_9] = A64_R(22), 51 [BPF_REG_9] = A64_R(22),
52 /* read-only frame pointer to access stack */ 52 /* read-only frame pointer to access stack */
53 [BPF_REG_FP] = A64_FP, 53 [BPF_REG_FP] = A64_R(25),
54 /* temporary register for internal BPF JIT */ 54 /* temporary register for internal BPF JIT */
55 [TMP_REG_1] = A64_R(23), 55 [TMP_REG_1] = A64_R(23),
56 [TMP_REG_2] = A64_R(24), 56 [TMP_REG_2] = A64_R(24),
@@ -155,18 +155,49 @@ static void build_prologue(struct jit_ctx *ctx)
155 stack_size += 4; /* extra for skb_copy_bits buffer */ 155 stack_size += 4; /* extra for skb_copy_bits buffer */
156 stack_size = STACK_ALIGN(stack_size); 156 stack_size = STACK_ALIGN(stack_size);
157 157
158 /*
159 * BPF prog stack layout
160 *
161 * high
162 * original A64_SP => 0:+-----+ BPF prologue
163 * |FP/LR|
164 * current A64_FP => -16:+-----+
165 * | ... | callee saved registers
166 * +-----+
167 * | | x25/x26
168 * BPF fp register => -80:+-----+
169 * | |
170 * | ... | BPF prog stack
171 * | |
172 * | |
173 * current A64_SP => +-----+
174 * | |
175 * | ... | Function call stack
176 * | |
177 * +-----+
178 * low
179 *
180 */
181
182 /* Save FP and LR registers to stay align with ARM64 AAPCS */
183 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
184 emit(A64_MOV(1, A64_FP, A64_SP), ctx);
185
158 /* Save callee-saved register */ 186 /* Save callee-saved register */
159 emit(A64_PUSH(r6, r7, A64_SP), ctx); 187 emit(A64_PUSH(r6, r7, A64_SP), ctx);
160 emit(A64_PUSH(r8, r9, A64_SP), ctx); 188 emit(A64_PUSH(r8, r9, A64_SP), ctx);
161 if (ctx->tmp_used) 189 if (ctx->tmp_used)
162 emit(A64_PUSH(tmp1, tmp2, A64_SP), ctx); 190 emit(A64_PUSH(tmp1, tmp2, A64_SP), ctx);
163 191
164 /* Set up BPF stack */ 192 /* Save fp (x25) and x26. SP requires 16 bytes alignment */
165 emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx); 193 emit(A64_PUSH(fp, A64_R(26), A64_SP), ctx);
166 194
167 /* Set up frame pointer */ 195 /* Set up BPF prog stack base register (x25) */
168 emit(A64_MOV(1, fp, A64_SP), ctx); 196 emit(A64_MOV(1, fp, A64_SP), ctx);
169 197
198 /* Set up function call stack */
199 emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx);
200
170 /* Clear registers A and X */ 201 /* Clear registers A and X */
171 emit_a64_mov_i64(ra, 0, ctx); 202 emit_a64_mov_i64(ra, 0, ctx);
172 emit_a64_mov_i64(rx, 0, ctx); 203 emit_a64_mov_i64(rx, 0, ctx);
@@ -190,14 +221,17 @@ static void build_epilogue(struct jit_ctx *ctx)
190 /* We're done with BPF stack */ 221 /* We're done with BPF stack */
191 emit(A64_ADD_I(1, A64_SP, A64_SP, stack_size), ctx); 222 emit(A64_ADD_I(1, A64_SP, A64_SP, stack_size), ctx);
192 223
224 /* Restore fs (x25) and x26 */
225 emit(A64_POP(fp, A64_R(26), A64_SP), ctx);
226
193 /* Restore callee-saved register */ 227 /* Restore callee-saved register */
194 if (ctx->tmp_used) 228 if (ctx->tmp_used)
195 emit(A64_POP(tmp1, tmp2, A64_SP), ctx); 229 emit(A64_POP(tmp1, tmp2, A64_SP), ctx);
196 emit(A64_POP(r8, r9, A64_SP), ctx); 230 emit(A64_POP(r8, r9, A64_SP), ctx);
197 emit(A64_POP(r6, r7, A64_SP), ctx); 231 emit(A64_POP(r6, r7, A64_SP), ctx);
198 232
199 /* Restore frame pointer */ 233 /* Restore FP/LR registers */
200 emit(A64_MOV(1, fp, A64_SP), ctx); 234 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
201 235
202 /* Set return value */ 236 /* Set return value */
203 emit(A64_MOV(1, A64_R(0), r0), ctx); 237 emit(A64_MOV(1, A64_R(0), r0), ctx);
@@ -225,6 +259,17 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
225 u8 jmp_cond; 259 u8 jmp_cond;
226 s32 jmp_offset; 260 s32 jmp_offset;
227 261
262#define check_imm(bits, imm) do { \
263 if ((((imm) > 0) && ((imm) >> (bits))) || \
264 (((imm) < 0) && (~(imm) >> (bits)))) { \
265 pr_info("[%2d] imm=%d(0x%x) out of range\n", \
266 i, imm, imm); \
267 return -EINVAL; \
268 } \
269} while (0)
270#define check_imm19(imm) check_imm(19, imm)
271#define check_imm26(imm) check_imm(26, imm)
272
228 switch (code) { 273 switch (code) {
229 /* dst = src */ 274 /* dst = src */
230 case BPF_ALU | BPF_MOV | BPF_X: 275 case BPF_ALU | BPF_MOV | BPF_X:
@@ -258,15 +303,33 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
258 break; 303 break;
259 case BPF_ALU | BPF_DIV | BPF_X: 304 case BPF_ALU | BPF_DIV | BPF_X:
260 case BPF_ALU64 | BPF_DIV | BPF_X: 305 case BPF_ALU64 | BPF_DIV | BPF_X:
261 emit(A64_UDIV(is64, dst, dst, src), ctx);
262 break;
263 case BPF_ALU | BPF_MOD | BPF_X: 306 case BPF_ALU | BPF_MOD | BPF_X:
264 case BPF_ALU64 | BPF_MOD | BPF_X: 307 case BPF_ALU64 | BPF_MOD | BPF_X:
265 ctx->tmp_used = 1; 308 {
266 emit(A64_UDIV(is64, tmp, dst, src), ctx); 309 const u8 r0 = bpf2a64[BPF_REG_0];
267 emit(A64_MUL(is64, tmp, tmp, src), ctx); 310
268 emit(A64_SUB(is64, dst, dst, tmp), ctx); 311 /* if (src == 0) return 0 */
312 jmp_offset = 3; /* skip ahead to else path */
313 check_imm19(jmp_offset);
314 emit(A64_CBNZ(is64, src, jmp_offset), ctx);
315 emit(A64_MOVZ(1, r0, 0, 0), ctx);
316 jmp_offset = epilogue_offset(ctx);
317 check_imm26(jmp_offset);
318 emit(A64_B(jmp_offset), ctx);
319 /* else */
320 switch (BPF_OP(code)) {
321 case BPF_DIV:
322 emit(A64_UDIV(is64, dst, dst, src), ctx);
323 break;
324 case BPF_MOD:
325 ctx->tmp_used = 1;
326 emit(A64_UDIV(is64, tmp, dst, src), ctx);
327 emit(A64_MUL(is64, tmp, tmp, src), ctx);
328 emit(A64_SUB(is64, dst, dst, tmp), ctx);
329 break;
330 }
269 break; 331 break;
332 }
270 case BPF_ALU | BPF_LSH | BPF_X: 333 case BPF_ALU | BPF_LSH | BPF_X:
271 case BPF_ALU64 | BPF_LSH | BPF_X: 334 case BPF_ALU64 | BPF_LSH | BPF_X:
272 emit(A64_LSLV(is64, dst, dst, src), ctx); 335 emit(A64_LSLV(is64, dst, dst, src), ctx);
@@ -393,17 +456,6 @@ emit_bswap_uxt:
393 emit(A64_ASR(is64, dst, dst, imm), ctx); 456 emit(A64_ASR(is64, dst, dst, imm), ctx);
394 break; 457 break;
395 458
396#define check_imm(bits, imm) do { \
397 if ((((imm) > 0) && ((imm) >> (bits))) || \
398 (((imm) < 0) && (~(imm) >> (bits)))) { \
399 pr_info("[%2d] imm=%d(0x%x) out of range\n", \
400 i, imm, imm); \
401 return -EINVAL; \
402 } \
403} while (0)
404#define check_imm19(imm) check_imm(19, imm)
405#define check_imm26(imm) check_imm(26, imm)
406
407 /* JUMP off */ 459 /* JUMP off */
408 case BPF_JMP | BPF_JA: 460 case BPF_JMP | BPF_JA:
409 jmp_offset = bpf2a64_offset(i + off, i, ctx); 461 jmp_offset = bpf2a64_offset(i + off, i, ctx);
@@ -740,11 +792,11 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
740 if (bpf_jit_enable > 1) 792 if (bpf_jit_enable > 1)
741 bpf_jit_dump(prog->len, image_size, 2, ctx.image); 793 bpf_jit_dump(prog->len, image_size, 2, ctx.image);
742 794
743 bpf_flush_icache(ctx.image, ctx.image + ctx.idx); 795 bpf_flush_icache(header, ctx.image + ctx.idx);
744 796
745 set_memory_ro((unsigned long)header, header->pages); 797 set_memory_ro((unsigned long)header, header->pages);
746 prog->bpf_func = (void *)ctx.image; 798 prog->bpf_func = (void *)ctx.image;
747 prog->jited = true; 799 prog->jited = 1;
748out: 800out:
749 kfree(ctx.offset); 801 kfree(ctx.offset);
750} 802}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-28 11:27:37 -0500