aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig31
-rw-r--r--lib/Kconfig.debug69
-rw-r--r--lib/Kconfig.kasan1
-rw-r--r--lib/Makefile26
-rw-r--r--lib/argv_split.c2
-rw-r--r--lib/bitmap.c5
-rw-r--r--lib/bucket_locks.c5
-rw-r--r--lib/dec_and_lock.c16
-rw-r--r--lib/dma-debug.c1752
-rw-r--r--lib/dma-direct.c185
-rw-r--r--lib/dma-virt.c61
-rw-r--r--lib/idr.c10
-rw-r--r--lib/interval_tree_test.c5
-rw-r--r--lib/iommu-common.c267
-rw-r--r--lib/iommu-helper.c14
-rw-r--r--lib/iov_iter.c65
-rw-r--r--lib/kfifo.c2
-rw-r--r--lib/kobject_uevent.c178
-rw-r--r--lib/lru_cache.c2
-rw-r--r--lib/mpi/mpi-internal.h75
-rw-r--r--lib/mpi/mpiutil.c4
-rw-r--r--lib/percpu_ida.c63
-rw-r--r--lib/radix-tree.c10
-rw-r--r--lib/rbtree_test.c2
-rw-r--r--lib/reed_solomon/decode_rs.c34
-rw-r--r--lib/reed_solomon/encode_rs.c15
-rw-r--r--lib/reed_solomon/reed_solomon.c240
-rw-r--r--lib/refcount.c28
-rw-r--r--lib/rhashtable.c51
-rw-r--r--lib/sbitmap.c115
-rw-r--r--lib/scatterlist.c9
-rw-r--r--lib/swiotlb.c1092
-rw-r--r--lib/test_bitmap.c21
-rw-r--r--lib/test_bpf.c615
-rw-r--r--lib/test_firmware.c10
-rw-r--r--lib/test_kmod.c5
-rw-r--r--lib/test_overflow.c417
-rw-r--r--lib/test_printf.c2
-rw-r--r--lib/test_rhashtable.c13
-rw-r--r--lib/ucmpdi2.c2
-rw-r--r--lib/ucs2_string.c2
-rw-r--r--lib/vsprintf.c155
42 files changed, 1579 insertions, 4097 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index 5fe577673b98..706836ec314d 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -405,7 +405,7 @@ config ASSOCIATIVE_ARRAY
405 405
406 See: 406 See:
407 407
408 Documentation/assoc_array.txt 408 Documentation/core-api/assoc_array.rst
409 409
410 for more information. 410 for more information.
411 411
@@ -420,24 +420,14 @@ config HAS_IOPORT_MAP
420 depends on HAS_IOMEM && !NO_IOPORT_MAP 420 depends on HAS_IOMEM && !NO_IOPORT_MAP
421 default y 421 default y
422 422
423config HAS_DMA 423source "kernel/dma/Kconfig"
424 bool
425 depends on !NO_DMA
426 default y
427 424
428config SGL_ALLOC 425config SGL_ALLOC
429 bool 426 bool
430 default n 427 default n
431 428
432config DMA_DIRECT_OPS 429config IOMMU_HELPER
433 bool
434 depends on HAS_DMA && (!64BIT || ARCH_DMA_ADDR_T_64BIT)
435 default n
436
437config DMA_VIRT_OPS
438 bool 430 bool
439 depends on HAS_DMA && (!64BIT || ARCH_DMA_ADDR_T_64BIT)
440 default n
441 431
442config CHECK_SIGNATURE 432config CHECK_SIGNATURE
443 bool 433 bool
@@ -586,6 +576,9 @@ config ARCH_HAS_PMEM_API
586config ARCH_HAS_UACCESS_FLUSHCACHE 576config ARCH_HAS_UACCESS_FLUSHCACHE
587 bool 577 bool
588 578
579config ARCH_HAS_UACCESS_MCSAFE
580 bool
581
589config STACKDEPOT 582config STACKDEPOT
590 bool 583 bool
591 select STACKTRACE 584 select STACKTRACE
@@ -604,20 +597,20 @@ config STRING_SELFTEST
604 597
605endmenu 598endmenu
606 599
607config GENERIC_ASHLDI3 600config GENERIC_LIB_ASHLDI3
608 bool 601 bool
609 602
610config GENERIC_ASHRDI3 603config GENERIC_LIB_ASHRDI3
611 bool 604 bool
612 605
613config GENERIC_LSHRDI3 606config GENERIC_LIB_LSHRDI3
614 bool 607 bool
615 608
616config GENERIC_MULDI3 609config GENERIC_LIB_MULDI3
617 bool 610 bool
618 611
619config GENERIC_CMPDI2 612config GENERIC_LIB_CMPDI2
620 bool 613 bool
621 614
622config GENERIC_UCMPDI2 615config GENERIC_LIB_UCMPDI2
623 bool 616 bool
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index c40c7b734cd1..8838d1158d19 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -736,12 +736,15 @@ config ARCH_HAS_KCOV
736 only for x86_64. KCOV requires testing on other archs, and most likely 736 only for x86_64. KCOV requires testing on other archs, and most likely
737 disabling of instrumentation for some early boot code. 737 disabling of instrumentation for some early boot code.
738 738
739config CC_HAS_SANCOV_TRACE_PC
740 def_bool $(cc-option,-fsanitize-coverage=trace-pc)
741
739config KCOV 742config KCOV
740 bool "Code coverage for fuzzing" 743 bool "Code coverage for fuzzing"
741 depends on ARCH_HAS_KCOV 744 depends on ARCH_HAS_KCOV
745 depends on CC_HAS_SANCOV_TRACE_PC || GCC_PLUGINS
742 select DEBUG_FS 746 select DEBUG_FS
743 select GCC_PLUGINS if !COMPILE_TEST 747 select GCC_PLUGIN_SANCOV if !CC_HAS_SANCOV_TRACE_PC
744 select GCC_PLUGIN_SANCOV if !COMPILE_TEST
745 help 748 help
746 KCOV exposes kernel code coverage information in a form suitable 749 KCOV exposes kernel code coverage information in a form suitable
747 for coverage-guided fuzzing (randomized testing). 750 for coverage-guided fuzzing (randomized testing).
@@ -755,7 +758,7 @@ config KCOV
755config KCOV_ENABLE_COMPARISONS 758config KCOV_ENABLE_COMPARISONS
756 bool "Enable comparison operands collection by KCOV" 759 bool "Enable comparison operands collection by KCOV"
757 depends on KCOV 760 depends on KCOV
758 default n 761 depends on $(cc-option,-fsanitize-coverage=trace-cmp)
759 help 762 help
760 KCOV also exposes operands of every comparison in the instrumented 763 KCOV also exposes operands of every comparison in the instrumented
761 code along with operand sizes and PCs of the comparison instructions. 764 code along with operand sizes and PCs of the comparison instructions.
@@ -765,7 +768,7 @@ config KCOV_ENABLE_COMPARISONS
765config KCOV_INSTRUMENT_ALL 768config KCOV_INSTRUMENT_ALL
766 bool "Instrument all code by default" 769 bool "Instrument all code by default"
767 depends on KCOV 770 depends on KCOV
768 default y if KCOV 771 default y
769 help 772 help
770 If you are doing generic system call fuzzing (like e.g. syzkaller), 773 If you are doing generic system call fuzzing (like e.g. syzkaller),
771 then you will want to instrument the whole kernel and you should 774 then you will want to instrument the whole kernel and you should
@@ -1503,6 +1506,10 @@ config NETDEV_NOTIFIER_ERROR_INJECT
1503 1506
1504 If unsure, say N. 1507 If unsure, say N.
1505 1508
1509config FUNCTION_ERROR_INJECTION
1510 def_bool y
1511 depends on HAVE_FUNCTION_ERROR_INJECTION && KPROBES
1512
1506config FAULT_INJECTION 1513config FAULT_INJECTION
1507 bool "Fault-injection framework" 1514 bool "Fault-injection framework"
1508 depends on DEBUG_KERNEL 1515 depends on DEBUG_KERNEL
@@ -1510,10 +1517,6 @@ config FAULT_INJECTION
1510 Provide fault-injection framework. 1517 Provide fault-injection framework.
1511 For more details, see Documentation/fault-injection/. 1518 For more details, see Documentation/fault-injection/.
1512 1519
1513config FUNCTION_ERROR_INJECTION
1514 def_bool y
1515 depends on HAVE_FUNCTION_ERROR_INJECTION && KPROBES
1516
1517config FAILSLAB 1520config FAILSLAB
1518 bool "Fault-injection capability for kmalloc" 1521 bool "Fault-injection capability for kmalloc"
1519 depends on FAULT_INJECTION 1522 depends on FAULT_INJECTION
@@ -1544,16 +1547,6 @@ config FAIL_IO_TIMEOUT
1544 Only works with drivers that use the generic timeout handling, 1547 Only works with drivers that use the generic timeout handling,
1545 for others it wont do anything. 1548 for others it wont do anything.
1546 1549
1547config FAIL_MMC_REQUEST
1548 bool "Fault-injection capability for MMC IO"
1549 depends on FAULT_INJECTION_DEBUG_FS && MMC
1550 help
1551 Provide fault-injection capability for MMC IO.
1552 This will make the mmc core return data errors. This is
1553 useful to test the error handling in the mmc block device
1554 and to test how the mmc host driver handles retries from
1555 the block device.
1556
1557config FAIL_FUTEX 1550config FAIL_FUTEX
1558 bool "Fault-injection capability for futexes" 1551 bool "Fault-injection capability for futexes"
1559 select DEBUG_FS 1552 select DEBUG_FS
@@ -1561,6 +1554,12 @@ config FAIL_FUTEX
1561 help 1554 help
1562 Provide fault-injection capability for futexes. 1555 Provide fault-injection capability for futexes.
1563 1556
1557config FAULT_INJECTION_DEBUG_FS
1558 bool "Debugfs entries for fault-injection capabilities"
1559 depends on FAULT_INJECTION && SYSFS && DEBUG_FS
1560 help
1561 Enable configuration of fault-injection capabilities via debugfs.
1562
1564config FAIL_FUNCTION 1563config FAIL_FUNCTION
1565 bool "Fault-injection capability for functions" 1564 bool "Fault-injection capability for functions"
1566 depends on FAULT_INJECTION_DEBUG_FS && FUNCTION_ERROR_INJECTION 1565 depends on FAULT_INJECTION_DEBUG_FS && FUNCTION_ERROR_INJECTION
@@ -1571,11 +1570,15 @@ config FAIL_FUNCTION
1571 an error value and have to handle it. This is useful to test the 1570 an error value and have to handle it. This is useful to test the
1572 error handling in various subsystems. 1571 error handling in various subsystems.
1573 1572
1574config FAULT_INJECTION_DEBUG_FS 1573config FAIL_MMC_REQUEST
1575 bool "Debugfs entries for fault-injection capabilities" 1574 bool "Fault-injection capability for MMC IO"
1576 depends on FAULT_INJECTION && SYSFS && DEBUG_FS 1575 depends on FAULT_INJECTION_DEBUG_FS && MMC
1577 help 1576 help
1578 Enable configuration of fault-injection capabilities via debugfs. 1577 Provide fault-injection capability for MMC IO.
1578 This will make the mmc core return data errors. This is
1579 useful to test the error handling in the mmc block device
1580 and to test how the mmc host driver handles retries from
1581 the block device.
1579 1582
1580config FAULT_INJECTION_STACKTRACE_FILTER 1583config FAULT_INJECTION_STACKTRACE_FILTER
1581 bool "stacktrace filter for fault-injection capabilities" 1584 bool "stacktrace filter for fault-injection capabilities"
@@ -1634,7 +1637,7 @@ config PROVIDE_OHCI1394_DMA_INIT
1634 1637
1635config DMA_API_DEBUG 1638config DMA_API_DEBUG
1636 bool "Enable debugging of DMA-API usage" 1639 bool "Enable debugging of DMA-API usage"
1637 depends on HAVE_DMA_API_DEBUG 1640 select NEED_DMA_MAP_STATE
1638 help 1641 help
1639 Enable this option to debug the use of the DMA API by device drivers. 1642 Enable this option to debug the use of the DMA API by device drivers.
1640 With this option you will be able to detect common bugs in device 1643 With this option you will be able to detect common bugs in device
@@ -1651,6 +1654,23 @@ config DMA_API_DEBUG
1651 1654
1652 If unsure, say N. 1655 If unsure, say N.
1653 1656
1657config DMA_API_DEBUG_SG
1658 bool "Debug DMA scatter-gather usage"
1659 default y
1660 depends on DMA_API_DEBUG
1661 help
1662 Perform extra checking that callers of dma_map_sg() have respected the
1663 appropriate segment length/boundary limits for the given device when
1664 preparing DMA scatterlists.
1665
1666 This is particularly likely to have been overlooked in cases where the
1667 dma_map_sg() API is used for general bulk mapping of pages rather than
1668 preparing literal scatter-gather descriptors, where there is a risk of
1669 unexpected behaviour from DMA API implementations if the scatterlist
1670 is technically out-of-spec.
1671
1672 If unsure, say N.
1673
1654menuconfig RUNTIME_TESTING_MENU 1674menuconfig RUNTIME_TESTING_MENU
1655 bool "Runtime Testing" 1675 bool "Runtime Testing"
1656 def_bool y 1676 def_bool y
@@ -1785,6 +1805,9 @@ config TEST_BITMAP
1785config TEST_UUID 1805config TEST_UUID
1786 tristate "Test functions located in the uuid module at runtime" 1806 tristate "Test functions located in the uuid module at runtime"
1787 1807
1808config TEST_OVERFLOW
1809 tristate "Test check_*_overflow() functions at runtime"
1810
1788config TEST_RHASHTABLE 1811config TEST_RHASHTABLE
1789 tristate "Perform selftest on resizable hash table" 1812 tristate "Perform selftest on resizable hash table"
1790 default n 1813 default n
diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan
index 3d35d062970d..c253c1b46c6b 100644
--- a/lib/Kconfig.kasan
+++ b/lib/Kconfig.kasan
@@ -6,6 +6,7 @@ if HAVE_ARCH_KASAN
6config KASAN 6config KASAN
7 bool "KASan: runtime memory debugger" 7 bool "KASan: runtime memory debugger"
8 depends on SLUB || (SLAB && !DEBUG_SLAB) 8 depends on SLUB || (SLAB && !DEBUG_SLAB)
9 select SLUB_DEBUG if SLUB
9 select CONSTRUCTORS 10 select CONSTRUCTORS
10 select STACKDEPOT 11 select STACKDEPOT
11 help 12 help
diff --git a/lib/Makefile b/lib/Makefile
index ce20696d5a92..90dc5520b784 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -23,14 +23,12 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \
23 sha1.o chacha20.o irq_regs.o argv_split.o \ 23 sha1.o chacha20.o irq_regs.o argv_split.o \
24 flex_proportions.o ratelimit.o show_mem.o \ 24 flex_proportions.o ratelimit.o show_mem.o \
25 is_single_threaded.o plist.o decompress.o kobject_uevent.o \ 25 is_single_threaded.o plist.o decompress.o kobject_uevent.o \
26 earlycpio.o seq_buf.o siphash.o \ 26 earlycpio.o seq_buf.o siphash.o dec_and_lock.o \
27 nmi_backtrace.o nodemask.o win_minmax.o 27 nmi_backtrace.o nodemask.o win_minmax.o
28 28
29lib-$(CONFIG_PRINTK) += dump_stack.o 29lib-$(CONFIG_PRINTK) += dump_stack.o
30lib-$(CONFIG_MMU) += ioremap.o 30lib-$(CONFIG_MMU) += ioremap.o
31lib-$(CONFIG_SMP) += cpumask.o 31lib-$(CONFIG_SMP) += cpumask.o
32lib-$(CONFIG_DMA_DIRECT_OPS) += dma-direct.o
33lib-$(CONFIG_DMA_VIRT_OPS) += dma-virt.o
34 32
35lib-y += kobject.o klist.o 33lib-y += kobject.o klist.o
36obj-y += lockref.o 34obj-y += lockref.o
@@ -59,6 +57,7 @@ UBSAN_SANITIZE_test_ubsan.o := y
59obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o 57obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
60obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o 58obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o
61obj-$(CONFIG_TEST_LKM) += test_module.o 59obj-$(CONFIG_TEST_LKM) += test_module.o
60obj-$(CONFIG_TEST_OVERFLOW) += test_overflow.o
62obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o 61obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o
63obj-$(CONFIG_TEST_SORT) += test_sort.o 62obj-$(CONFIG_TEST_SORT) += test_sort.o
64obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o 63obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o
@@ -96,10 +95,6 @@ obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o
96obj-$(CONFIG_DEBUG_LIST) += list_debug.o 95obj-$(CONFIG_DEBUG_LIST) += list_debug.o
97obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o 96obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o
98 97
99ifneq ($(CONFIG_HAVE_DEC_LOCK),y)
100 lib-y += dec_and_lock.o
101endif
102
103obj-$(CONFIG_BITREVERSE) += bitrev.o 98obj-$(CONFIG_BITREVERSE) += bitrev.o
104obj-$(CONFIG_RATIONAL) += rational.o 99obj-$(CONFIG_RATIONAL) += rational.o
105obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o 100obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o
@@ -146,8 +141,7 @@ obj-$(CONFIG_SMP) += percpu_counter.o
146obj-$(CONFIG_AUDIT_GENERIC) += audit.o 141obj-$(CONFIG_AUDIT_GENERIC) += audit.o
147obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o 142obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o
148 143
149obj-$(CONFIG_SWIOTLB) += swiotlb.o 144obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o
150obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o iommu-common.o
151obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o 145obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o
152obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o 146obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o
153obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o 147obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o
@@ -167,8 +161,6 @@ obj-$(CONFIG_NLATTR) += nlattr.o
167 161
168obj-$(CONFIG_LRU_CACHE) += lru_cache.o 162obj-$(CONFIG_LRU_CACHE) += lru_cache.o
169 163
170obj-$(CONFIG_DMA_API_DEBUG) += dma-debug.o
171
172obj-$(CONFIG_GENERIC_CSUM) += checksum.o 164obj-$(CONFIG_GENERIC_CSUM) += checksum.o
173 165
174obj-$(CONFIG_GENERIC_ATOMIC64) += atomic64.o 166obj-$(CONFIG_GENERIC_ATOMIC64) += atomic64.o
@@ -259,9 +251,9 @@ obj-$(CONFIG_SBITMAP) += sbitmap.o
259obj-$(CONFIG_PARMAN) += parman.o 251obj-$(CONFIG_PARMAN) += parman.o
260 252
261# GCC library routines 253# GCC library routines
262obj-$(CONFIG_GENERIC_ASHLDI3) += ashldi3.o 254obj-$(CONFIG_GENERIC_LIB_ASHLDI3) += ashldi3.o
263obj-$(CONFIG_GENERIC_ASHRDI3) += ashrdi3.o 255obj-$(CONFIG_GENERIC_LIB_ASHRDI3) += ashrdi3.o
264obj-$(CONFIG_GENERIC_LSHRDI3) += lshrdi3.o 256obj-$(CONFIG_GENERIC_LIB_LSHRDI3) += lshrdi3.o
265obj-$(CONFIG_GENERIC_MULDI3) += muldi3.o 257obj-$(CONFIG_GENERIC_LIB_MULDI3) += muldi3.o
266obj-$(CONFIG_GENERIC_CMPDI2) += cmpdi2.o 258obj-$(CONFIG_GENERIC_LIB_CMPDI2) += cmpdi2.o
267obj-$(CONFIG_GENERIC_UCMPDI2) += ucmpdi2.o 259obj-$(CONFIG_GENERIC_LIB_UCMPDI2) += ucmpdi2.o
diff --git a/lib/argv_split.c b/lib/argv_split.c
index 5c35752a9414..1a19a0a93dc1 100644
--- a/lib/argv_split.c
+++ b/lib/argv_split.c
@@ -69,7 +69,7 @@ char **argv_split(gfp_t gfp, const char *str, int *argcp)
69 return NULL; 69 return NULL;
70 70
71 argc = count_argc(argv_str); 71 argc = count_argc(argv_str);
72 argv = kmalloc(sizeof(*argv) * (argc + 2), gfp); 72 argv = kmalloc_array(argc + 2, sizeof(*argv), gfp);
73 if (!argv) { 73 if (!argv) {
74 kfree(argv_str); 74 kfree(argv_str);
75 return NULL; 75 return NULL;
diff --git a/lib/bitmap.c b/lib/bitmap.c
index a42eff7e8c48..58f9750e49c6 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -64,12 +64,9 @@ EXPORT_SYMBOL(__bitmap_equal);
64 64
65void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits) 65void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits)
66{ 66{
67 unsigned int k, lim = bits/BITS_PER_LONG; 67 unsigned int k, lim = BITS_TO_LONGS(bits);
68 for (k = 0; k < lim; ++k) 68 for (k = 0; k < lim; ++k)
69 dst[k] = ~src[k]; 69 dst[k] = ~src[k];
70
71 if (bits % BITS_PER_LONG)
72 dst[k] = ~src[k];
73} 70}
74EXPORT_SYMBOL(__bitmap_complement); 71EXPORT_SYMBOL(__bitmap_complement);
75 72
diff --git a/lib/bucket_locks.c b/lib/bucket_locks.c
index 266a97c5708b..ade3ce6c4af6 100644
--- a/lib/bucket_locks.c
+++ b/lib/bucket_locks.c
@@ -30,10 +30,7 @@ int alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *locks_mask,
30 } 30 }
31 31
32 if (sizeof(spinlock_t) != 0) { 32 if (sizeof(spinlock_t) != 0) {
33 if (gfpflags_allow_blocking(gfp)) 33 tlocks = kvmalloc_array(size, sizeof(spinlock_t), gfp);
34 tlocks = kvmalloc(size * sizeof(spinlock_t), gfp);
35 else
36 tlocks = kmalloc_array(size, sizeof(spinlock_t), gfp);
37 if (!tlocks) 34 if (!tlocks)
38 return -ENOMEM; 35 return -ENOMEM;
39 for (i = 0; i < size; i++) 36 for (i = 0; i < size; i++)
diff --git a/lib/dec_and_lock.c b/lib/dec_and_lock.c
index 347fa7ac2e8a..9555b68bb774 100644
--- a/lib/dec_and_lock.c
+++ b/lib/dec_and_lock.c
@@ -33,3 +33,19 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
33} 33}
34 34
35EXPORT_SYMBOL(_atomic_dec_and_lock); 35EXPORT_SYMBOL(_atomic_dec_and_lock);
36
37int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
38 unsigned long *flags)
39{
40 /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
41 if (atomic_add_unless(atomic, -1, 1))
42 return 0;
43
44 /* Otherwise do it the slow way */
45 spin_lock_irqsave(lock, *flags);
46 if (atomic_dec_and_test(atomic))
47 return 1;
48 spin_unlock_irqrestore(lock, *flags);
49 return 0;
50}
51EXPORT_SYMBOL(_atomic_dec_and_lock_irqsave);
diff --git a/lib/dma-debug.c b/lib/dma-debug.c
deleted file mode 100644
index 7f5cdc1e6b29..000000000000
--- a/lib/dma-debug.c
+++ /dev/null
@@ -1,1752 +0,0 @@
1/*
2 * Copyright (C) 2008 Advanced Micro Devices, Inc.
3 *
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20#include <linux/sched/task_stack.h>
21#include <linux/scatterlist.h>
22#include <linux/dma-mapping.h>
23#include <linux/sched/task.h>
24#include <linux/stacktrace.h>
25#include <linux/dma-debug.h>
26#include <linux/spinlock.h>
27#include <linux/vmalloc.h>
28#include <linux/debugfs.h>
29#include <linux/uaccess.h>
30#include <linux/export.h>
31#include <linux/device.h>
32#include <linux/types.h>
33#include <linux/sched.h>
34#include <linux/ctype.h>
35#include <linux/list.h>
36#include <linux/slab.h>
37
38#include <asm/sections.h>
39
40#define HASH_SIZE 1024ULL
41#define HASH_FN_SHIFT 13
42#define HASH_FN_MASK (HASH_SIZE - 1)
43
44enum {
45 dma_debug_single,
46 dma_debug_page,
47 dma_debug_sg,
48 dma_debug_coherent,
49 dma_debug_resource,
50};
51
52enum map_err_types {
53 MAP_ERR_CHECK_NOT_APPLICABLE,
54 MAP_ERR_NOT_CHECKED,
55 MAP_ERR_CHECKED,
56};
57
58#define DMA_DEBUG_STACKTRACE_ENTRIES 5
59
60/**
61 * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
62 * @list: node on pre-allocated free_entries list
63 * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
64 * @type: single, page, sg, coherent
65 * @pfn: page frame of the start address
66 * @offset: offset of mapping relative to pfn
67 * @size: length of the mapping
68 * @direction: enum dma_data_direction
69 * @sg_call_ents: 'nents' from dma_map_sg
70 * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
71 * @map_err_type: track whether dma_mapping_error() was checked
72 * @stacktrace: support backtraces when a violation is detected
73 */
74struct dma_debug_entry {
75 struct list_head list;
76 struct device *dev;
77 int type;
78 unsigned long pfn;
79 size_t offset;
80 u64 dev_addr;
81 u64 size;
82 int direction;
83 int sg_call_ents;
84 int sg_mapped_ents;
85 enum map_err_types map_err_type;
86#ifdef CONFIG_STACKTRACE
87 struct stack_trace stacktrace;
88 unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
89#endif
90};
91
92typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
93
94struct hash_bucket {
95 struct list_head list;
96 spinlock_t lock;
97} ____cacheline_aligned_in_smp;
98
99/* Hash list to save the allocated dma addresses */
100static struct hash_bucket dma_entry_hash[HASH_SIZE];
101/* List of pre-allocated dma_debug_entry's */
102static LIST_HEAD(free_entries);
103/* Lock for the list above */
104static DEFINE_SPINLOCK(free_entries_lock);
105
106/* Global disable flag - will be set in case of an error */
107static bool global_disable __read_mostly;
108
109/* Early initialization disable flag, set at the end of dma_debug_init */
110static bool dma_debug_initialized __read_mostly;
111
112static inline bool dma_debug_disabled(void)
113{
114 return global_disable || !dma_debug_initialized;
115}
116
117/* Global error count */
118static u32 error_count;
119
120/* Global error show enable*/
121static u32 show_all_errors __read_mostly;
122/* Number of errors to show */
123static u32 show_num_errors = 1;
124
125static u32 num_free_entries;
126static u32 min_free_entries;
127static u32 nr_total_entries;
128
129/* number of preallocated entries requested by kernel cmdline */
130static u32 req_entries;
131
132/* debugfs dentry's for the stuff above */
133static struct dentry *dma_debug_dent __read_mostly;
134static struct dentry *global_disable_dent __read_mostly;
135static struct dentry *error_count_dent __read_mostly;
136static struct dentry *show_all_errors_dent __read_mostly;
137static struct dentry *show_num_errors_dent __read_mostly;
138static struct dentry *num_free_entries_dent __read_mostly;
139static struct dentry *min_free_entries_dent __read_mostly;
140static struct dentry *filter_dent __read_mostly;
141
142/* per-driver filter related state */
143
144#define NAME_MAX_LEN 64
145
146static char current_driver_name[NAME_MAX_LEN] __read_mostly;
147static struct device_driver *current_driver __read_mostly;
148
149static DEFINE_RWLOCK(driver_name_lock);
150
151static const char *const maperr2str[] = {
152 [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
153 [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
154 [MAP_ERR_CHECKED] = "dma map error checked",
155};
156
157static const char *type2name[5] = { "single", "page",
158 "scather-gather", "coherent",
159 "resource" };
160
161static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
162 "DMA_FROM_DEVICE", "DMA_NONE" };
163
164/*
165 * The access to some variables in this macro is racy. We can't use atomic_t
166 * here because all these variables are exported to debugfs. Some of them even
167 * writeable. This is also the reason why a lock won't help much. But anyway,
168 * the races are no big deal. Here is why:
169 *
170 * error_count: the addition is racy, but the worst thing that can happen is
171 * that we don't count some errors
172 * show_num_errors: the subtraction is racy. Also no big deal because in
173 * worst case this will result in one warning more in the
174 * system log than the user configured. This variable is
175 * writeable via debugfs.
176 */
177static inline void dump_entry_trace(struct dma_debug_entry *entry)
178{
179#ifdef CONFIG_STACKTRACE
180 if (entry) {
181 pr_warning("Mapped at:\n");
182 print_stack_trace(&entry->stacktrace, 0);
183 }
184#endif
185}
186
187static bool driver_filter(struct device *dev)
188{
189 struct device_driver *drv;
190 unsigned long flags;
191 bool ret;
192
193 /* driver filter off */
194 if (likely(!current_driver_name[0]))
195 return true;
196
197 /* driver filter on and initialized */
198 if (current_driver && dev && dev->driver == current_driver)
199 return true;
200
201 /* driver filter on, but we can't filter on a NULL device... */
202 if (!dev)
203 return false;
204
205 if (current_driver || !current_driver_name[0])
206 return false;
207
208 /* driver filter on but not yet initialized */
209 drv = dev->driver;
210 if (!drv)
211 return false;
212
213 /* lock to protect against change of current_driver_name */
214 read_lock_irqsave(&driver_name_lock, flags);
215
216 ret = false;
217 if (drv->name &&
218 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
219 current_driver = drv;
220 ret = true;
221 }
222
223 read_unlock_irqrestore(&driver_name_lock, flags);
224
225 return ret;
226}
227
228#define err_printk(dev, entry, format, arg...) do { \
229 error_count += 1; \
230 if (driver_filter(dev) && \
231 (show_all_errors || show_num_errors > 0)) { \
232 WARN(1, "%s %s: " format, \
233 dev ? dev_driver_string(dev) : "NULL", \
234 dev ? dev_name(dev) : "NULL", ## arg); \
235 dump_entry_trace(entry); \
236 } \
237 if (!show_all_errors && show_num_errors > 0) \
238 show_num_errors -= 1; \
239 } while (0);
240
241/*
242 * Hash related functions
243 *
244 * Every DMA-API request is saved into a struct dma_debug_entry. To
245 * have quick access to these structs they are stored into a hash.
246 */
247static int hash_fn(struct dma_debug_entry *entry)
248{
249 /*
250 * Hash function is based on the dma address.
251 * We use bits 20-27 here as the index into the hash
252 */
253 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
254}
255
256/*
257 * Request exclusive access to a hash bucket for a given dma_debug_entry.
258 */
259static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
260 unsigned long *flags)
261 __acquires(&dma_entry_hash[idx].lock)
262{
263 int idx = hash_fn(entry);
264 unsigned long __flags;
265
266 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
267 *flags = __flags;
268 return &dma_entry_hash[idx];
269}
270
271/*
272 * Give up exclusive access to the hash bucket
273 */
274static void put_hash_bucket(struct hash_bucket *bucket,
275 unsigned long *flags)
276 __releases(&bucket->lock)
277{
278 unsigned long __flags = *flags;
279
280 spin_unlock_irqrestore(&bucket->lock, __flags);
281}
282
283static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
284{
285 return ((a->dev_addr == b->dev_addr) &&
286 (a->dev == b->dev)) ? true : false;
287}
288
289static bool containing_match(struct dma_debug_entry *a,
290 struct dma_debug_entry *b)
291{
292 if (a->dev != b->dev)
293 return false;
294
295 if ((b->dev_addr <= a->dev_addr) &&
296 ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
297 return true;
298
299 return false;
300}
301
302/*
303 * Search a given entry in the hash bucket list
304 */
305static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
306 struct dma_debug_entry *ref,
307 match_fn match)
308{
309 struct dma_debug_entry *entry, *ret = NULL;
310 int matches = 0, match_lvl, last_lvl = -1;
311
312 list_for_each_entry(entry, &bucket->list, list) {
313 if (!match(ref, entry))
314 continue;
315
316 /*
317 * Some drivers map the same physical address multiple
318 * times. Without a hardware IOMMU this results in the
319 * same device addresses being put into the dma-debug
320 * hash multiple times too. This can result in false
321 * positives being reported. Therefore we implement a
322 * best-fit algorithm here which returns the entry from
323 * the hash which fits best to the reference value
324 * instead of the first-fit.
325 */
326 matches += 1;
327 match_lvl = 0;
328 entry->size == ref->size ? ++match_lvl : 0;
329 entry->type == ref->type ? ++match_lvl : 0;
330 entry->direction == ref->direction ? ++match_lvl : 0;
331 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
332
333 if (match_lvl == 4) {
334 /* perfect-fit - return the result */
335 return entry;
336 } else if (match_lvl > last_lvl) {
337 /*
338 * We found an entry that fits better then the
339 * previous one or it is the 1st match.
340 */
341 last_lvl = match_lvl;
342 ret = entry;
343 }
344 }
345
346 /*
347 * If we have multiple matches but no perfect-fit, just return
348 * NULL.
349 */
350 ret = (matches == 1) ? ret : NULL;
351
352 return ret;
353}
354
355static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
356 struct dma_debug_entry *ref)
357{
358 return __hash_bucket_find(bucket, ref, exact_match);
359}
360
361static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
362 struct dma_debug_entry *ref,
363 unsigned long *flags)
364{
365
366 unsigned int max_range = dma_get_max_seg_size(ref->dev);
367 struct dma_debug_entry *entry, index = *ref;
368 unsigned int range = 0;
369
370 while (range <= max_range) {
371 entry = __hash_bucket_find(*bucket, ref, containing_match);
372
373 if (entry)
374 return entry;
375
376 /*
377 * Nothing found, go back a hash bucket
378 */
379 put_hash_bucket(*bucket, flags);
380 range += (1 << HASH_FN_SHIFT);
381 index.dev_addr -= (1 << HASH_FN_SHIFT);
382 *bucket = get_hash_bucket(&index, flags);
383 }
384
385 return NULL;
386}
387
388/*
389 * Add an entry to a hash bucket
390 */
391static void hash_bucket_add(struct hash_bucket *bucket,
392 struct dma_debug_entry *entry)
393{
394 list_add_tail(&entry->list, &bucket->list);
395}
396
397/*
398 * Remove entry from a hash bucket list
399 */
400static void hash_bucket_del(struct dma_debug_entry *entry)
401{
402 list_del(&entry->list);
403}
404
405static unsigned long long phys_addr(struct dma_debug_entry *entry)
406{
407 if (entry->type == dma_debug_resource)
408 return __pfn_to_phys(entry->pfn) + entry->offset;
409
410 return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
411}
412
413/*
414 * Dump mapping entries for debugging purposes
415 */
416void debug_dma_dump_mappings(struct device *dev)
417{
418 int idx;
419
420 for (idx = 0; idx < HASH_SIZE; idx++) {
421 struct hash_bucket *bucket = &dma_entry_hash[idx];
422 struct dma_debug_entry *entry;
423 unsigned long flags;
424
425 spin_lock_irqsave(&bucket->lock, flags);
426
427 list_for_each_entry(entry, &bucket->list, list) {
428 if (!dev || dev == entry->dev) {
429 dev_info(entry->dev,
430 "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
431 type2name[entry->type], idx,
432 phys_addr(entry), entry->pfn,
433 entry->dev_addr, entry->size,
434 dir2name[entry->direction],
435 maperr2str[entry->map_err_type]);
436 }
437 }
438
439 spin_unlock_irqrestore(&bucket->lock, flags);
440 }
441}
442EXPORT_SYMBOL(debug_dma_dump_mappings);
443
444/*
445 * For each mapping (initial cacheline in the case of
446 * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
447 * scatterlist, or the cacheline specified in dma_map_single) insert
448 * into this tree using the cacheline as the key. At
449 * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If
450 * the entry already exists at insertion time add a tag as a reference
451 * count for the overlapping mappings. For now, the overlap tracking
452 * just ensures that 'unmaps' balance 'maps' before marking the
453 * cacheline idle, but we should also be flagging overlaps as an API
454 * violation.
455 *
456 * Memory usage is mostly constrained by the maximum number of available
457 * dma-debug entries in that we need a free dma_debug_entry before
458 * inserting into the tree. In the case of dma_map_page and
459 * dma_alloc_coherent there is only one dma_debug_entry and one
460 * dma_active_cacheline entry to track per event. dma_map_sg(), on the
461 * other hand, consumes a single dma_debug_entry, but inserts 'nents'
462 * entries into the tree.
463 *
464 * At any time debug_dma_assert_idle() can be called to trigger a
465 * warning if any cachelines in the given page are in the active set.
466 */
467static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
468static DEFINE_SPINLOCK(radix_lock);
469#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
470#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
471#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
472
473static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
474{
475 return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
476 (entry->offset >> L1_CACHE_SHIFT);
477}
478
479static int active_cacheline_read_overlap(phys_addr_t cln)
480{
481 int overlap = 0, i;
482
483 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
484 if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
485 overlap |= 1 << i;
486 return overlap;
487}
488
489static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
490{
491 int i;
492
493 if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
494 return overlap;
495
496 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
497 if (overlap & 1 << i)
498 radix_tree_tag_set(&dma_active_cacheline, cln, i);
499 else
500 radix_tree_tag_clear(&dma_active_cacheline, cln, i);
501
502 return overlap;
503}
504
505static void active_cacheline_inc_overlap(phys_addr_t cln)
506{
507 int overlap = active_cacheline_read_overlap(cln);
508
509 overlap = active_cacheline_set_overlap(cln, ++overlap);
510
511 /* If we overflowed the overlap counter then we're potentially
512 * leaking dma-mappings. Otherwise, if maps and unmaps are
513 * balanced then this overflow may cause false negatives in
514 * debug_dma_assert_idle() as the cacheline may be marked idle
515 * prematurely.
516 */
517 WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
518 "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
519 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
520}
521
522static int active_cacheline_dec_overlap(phys_addr_t cln)
523{
524 int overlap = active_cacheline_read_overlap(cln);
525
526 return active_cacheline_set_overlap(cln, --overlap);
527}
528
529static int active_cacheline_insert(struct dma_debug_entry *entry)
530{
531 phys_addr_t cln = to_cacheline_number(entry);
532 unsigned long flags;
533 int rc;
534
535 /* If the device is not writing memory then we don't have any
536 * concerns about the cpu consuming stale data. This mitigates
537 * legitimate usages of overlapping mappings.
538 */
539 if (entry->direction == DMA_TO_DEVICE)
540 return 0;
541
542 spin_lock_irqsave(&radix_lock, flags);
543 rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
544 if (rc == -EEXIST)
545 active_cacheline_inc_overlap(cln);
546 spin_unlock_irqrestore(&radix_lock, flags);
547
548 return rc;
549}
550
551static void active_cacheline_remove(struct dma_debug_entry *entry)
552{
553 phys_addr_t cln = to_cacheline_number(entry);
554 unsigned long flags;
555
556 /* ...mirror the insert case */
557 if (entry->direction == DMA_TO_DEVICE)
558 return;
559
560 spin_lock_irqsave(&radix_lock, flags);
561 /* since we are counting overlaps the final put of the
562 * cacheline will occur when the overlap count is 0.
563 * active_cacheline_dec_overlap() returns -1 in that case
564 */
565 if (active_cacheline_dec_overlap(cln) < 0)
566 radix_tree_delete(&dma_active_cacheline, cln);
567 spin_unlock_irqrestore(&radix_lock, flags);
568}
569
570/**
571 * debug_dma_assert_idle() - assert that a page is not undergoing dma
572 * @page: page to lookup in the dma_active_cacheline tree
573 *
574 * Place a call to this routine in cases where the cpu touching the page
575 * before the dma completes (page is dma_unmapped) will lead to data
576 * corruption.
577 */
578void debug_dma_assert_idle(struct page *page)
579{
580 static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
581 struct dma_debug_entry *entry = NULL;
582 void **results = (void **) &ents;
583 unsigned int nents, i;
584 unsigned long flags;
585 phys_addr_t cln;
586
587 if (dma_debug_disabled())
588 return;
589
590 if (!page)
591 return;
592
593 cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
594 spin_lock_irqsave(&radix_lock, flags);
595 nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
596 CACHELINES_PER_PAGE);
597 for (i = 0; i < nents; i++) {
598 phys_addr_t ent_cln = to_cacheline_number(ents[i]);
599
600 if (ent_cln == cln) {
601 entry = ents[i];
602 break;
603 } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
604 break;
605 }
606 spin_unlock_irqrestore(&radix_lock, flags);
607
608 if (!entry)
609 return;
610
611 cln = to_cacheline_number(entry);
612 err_printk(entry->dev, entry,
613 "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
614 &cln);
615}
616
617/*
618 * Wrapper function for adding an entry to the hash.
619 * This function takes care of locking itself.
620 */
621static void add_dma_entry(struct dma_debug_entry *entry)
622{
623 struct hash_bucket *bucket;
624 unsigned long flags;
625 int rc;
626
627 bucket = get_hash_bucket(entry, &flags);
628 hash_bucket_add(bucket, entry);
629 put_hash_bucket(bucket, &flags);
630
631 rc = active_cacheline_insert(entry);
632 if (rc == -ENOMEM) {
633 pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
634 global_disable = true;
635 }
636
637 /* TODO: report -EEXIST errors here as overlapping mappings are
638 * not supported by the DMA API
639 */
640}
641
642static struct dma_debug_entry *__dma_entry_alloc(void)
643{
644 struct dma_debug_entry *entry;
645
646 entry = list_entry(free_entries.next, struct dma_debug_entry, list);
647 list_del(&entry->list);
648 memset(entry, 0, sizeof(*entry));
649
650 num_free_entries -= 1;
651 if (num_free_entries < min_free_entries)
652 min_free_entries = num_free_entries;
653
654 return entry;
655}
656
657/* struct dma_entry allocator
658 *
659 * The next two functions implement the allocator for
660 * struct dma_debug_entries.
661 */
662static struct dma_debug_entry *dma_entry_alloc(void)
663{
664 struct dma_debug_entry *entry;
665 unsigned long flags;
666
667 spin_lock_irqsave(&free_entries_lock, flags);
668
669 if (list_empty(&free_entries)) {
670 global_disable = true;
671 spin_unlock_irqrestore(&free_entries_lock, flags);
672 pr_err("DMA-API: debugging out of memory - disabling\n");
673 return NULL;
674 }
675
676 entry = __dma_entry_alloc();
677
678 spin_unlock_irqrestore(&free_entries_lock, flags);
679
680#ifdef CONFIG_STACKTRACE
681 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
682 entry->stacktrace.entries = entry->st_entries;
683 entry->stacktrace.skip = 2;
684 save_stack_trace(&entry->stacktrace);
685#endif
686
687 return entry;
688}
689
690static void dma_entry_free(struct dma_debug_entry *entry)
691{
692 unsigned long flags;
693
694 active_cacheline_remove(entry);
695
696 /*
697 * add to beginning of the list - this way the entries are
698 * more likely cache hot when they are reallocated.
699 */
700 spin_lock_irqsave(&free_entries_lock, flags);
701 list_add(&entry->list, &free_entries);
702 num_free_entries += 1;
703 spin_unlock_irqrestore(&free_entries_lock, flags);
704}
705
706int dma_debug_resize_entries(u32 num_entries)
707{
708 int i, delta, ret = 0;
709 unsigned long flags;
710 struct dma_debug_entry *entry;
711 LIST_HEAD(tmp);
712
713 spin_lock_irqsave(&free_entries_lock, flags);
714
715 if (nr_total_entries < num_entries) {
716 delta = num_entries - nr_total_entries;
717
718 spin_unlock_irqrestore(&free_entries_lock, flags);
719
720 for (i = 0; i < delta; i++) {
721 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
722 if (!entry)
723 break;
724
725 list_add_tail(&entry->list, &tmp);
726 }
727
728 spin_lock_irqsave(&free_entries_lock, flags);
729
730 list_splice(&tmp, &free_entries);
731 nr_total_entries += i;
732 num_free_entries += i;
733 } else {
734 delta = nr_total_entries - num_entries;
735
736 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
737 entry = __dma_entry_alloc();
738 kfree(entry);
739 }
740
741 nr_total_entries -= i;
742 }
743
744 if (nr_total_entries != num_entries)
745 ret = 1;
746
747 spin_unlock_irqrestore(&free_entries_lock, flags);
748
749 return ret;
750}
751EXPORT_SYMBOL(dma_debug_resize_entries);
752
753/*
754 * DMA-API debugging init code
755 *
756 * The init code does two things:
757 * 1. Initialize core data structures
758 * 2. Preallocate a given number of dma_debug_entry structs
759 */
760
761static int prealloc_memory(u32 num_entries)
762{
763 struct dma_debug_entry *entry, *next_entry;
764 int i;
765
766 for (i = 0; i < num_entries; ++i) {
767 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
768 if (!entry)
769 goto out_err;
770
771 list_add_tail(&entry->list, &free_entries);
772 }
773
774 num_free_entries = num_entries;
775 min_free_entries = num_entries;
776
777 pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
778
779 return 0;
780
781out_err:
782
783 list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
784 list_del(&entry->list);
785 kfree(entry);
786 }
787
788 return -ENOMEM;
789}
790
791static ssize_t filter_read(struct file *file, char __user *user_buf,
792 size_t count, loff_t *ppos)
793{
794 char buf[NAME_MAX_LEN + 1];
795 unsigned long flags;
796 int len;
797
798 if (!current_driver_name[0])
799 return 0;
800
801 /*
802 * We can't copy to userspace directly because current_driver_name can
803 * only be read under the driver_name_lock with irqs disabled. So
804 * create a temporary copy first.
805 */
806 read_lock_irqsave(&driver_name_lock, flags);
807 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
808 read_unlock_irqrestore(&driver_name_lock, flags);
809
810 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
811}
812
813static ssize_t filter_write(struct file *file, const char __user *userbuf,
814 size_t count, loff_t *ppos)
815{
816 char buf[NAME_MAX_LEN];
817 unsigned long flags;
818 size_t len;
819 int i;
820
821 /*
822 * We can't copy from userspace directly. Access to
823 * current_driver_name is protected with a write_lock with irqs
824 * disabled. Since copy_from_user can fault and may sleep we
825 * need to copy to temporary buffer first
826 */
827 len = min(count, (size_t)(NAME_MAX_LEN - 1));
828 if (copy_from_user(buf, userbuf, len))
829 return -EFAULT;
830
831 buf[len] = 0;
832
833 write_lock_irqsave(&driver_name_lock, flags);
834
835 /*
836 * Now handle the string we got from userspace very carefully.
837 * The rules are:
838 * - only use the first token we got
839 * - token delimiter is everything looking like a space
840 * character (' ', '\n', '\t' ...)
841 *
842 */
843 if (!isalnum(buf[0])) {
844 /*
845 * If the first character userspace gave us is not
846 * alphanumerical then assume the filter should be
847 * switched off.
848 */
849 if (current_driver_name[0])
850 pr_info("DMA-API: switching off dma-debug driver filter\n");
851 current_driver_name[0] = 0;
852 current_driver = NULL;
853 goto out_unlock;
854 }
855
856 /*
857 * Now parse out the first token and use it as the name for the
858 * driver to filter for.
859 */
860 for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
861 current_driver_name[i] = buf[i];
862 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
863 break;
864 }
865 current_driver_name[i] = 0;
866 current_driver = NULL;
867
868 pr_info("DMA-API: enable driver filter for driver [%s]\n",
869 current_driver_name);
870
871out_unlock:
872 write_unlock_irqrestore(&driver_name_lock, flags);
873
874 return count;
875}
876
877static const struct file_operations filter_fops = {
878 .read = filter_read,
879 .write = filter_write,
880 .llseek = default_llseek,
881};
882
883static int dma_debug_fs_init(void)
884{
885 dma_debug_dent = debugfs_create_dir("dma-api", NULL);
886 if (!dma_debug_dent) {
887 pr_err("DMA-API: can not create debugfs directory\n");
888 return -ENOMEM;
889 }
890
891 global_disable_dent = debugfs_create_bool("disabled", 0444,
892 dma_debug_dent,
893 &global_disable);
894 if (!global_disable_dent)
895 goto out_err;
896
897 error_count_dent = debugfs_create_u32("error_count", 0444,
898 dma_debug_dent, &error_count);
899 if (!error_count_dent)
900 goto out_err;
901
902 show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
903 dma_debug_dent,
904 &show_all_errors);
905 if (!show_all_errors_dent)
906 goto out_err;
907
908 show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
909 dma_debug_dent,
910 &show_num_errors);
911 if (!show_num_errors_dent)
912 goto out_err;
913
914 num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
915 dma_debug_dent,
916 &num_free_entries);
917 if (!num_free_entries_dent)
918 goto out_err;
919
920 min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
921 dma_debug_dent,
922 &min_free_entries);
923 if (!min_free_entries_dent)
924 goto out_err;
925
926 filter_dent = debugfs_create_file("driver_filter", 0644,
927 dma_debug_dent, NULL, &filter_fops);
928 if (!filter_dent)
929 goto out_err;
930
931 return 0;
932
933out_err:
934 debugfs_remove_recursive(dma_debug_dent);
935
936 return -ENOMEM;
937}
938
939static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
940{
941 struct dma_debug_entry *entry;
942 unsigned long flags;
943 int count = 0, i;
944
945 for (i = 0; i < HASH_SIZE; ++i) {
946 spin_lock_irqsave(&dma_entry_hash[i].lock, flags);
947 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
948 if (entry->dev == dev) {
949 count += 1;
950 *out_entry = entry;
951 }
952 }
953 spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags);
954 }
955
956 return count;
957}
958
959static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
960{
961 struct device *dev = data;
962 struct dma_debug_entry *uninitialized_var(entry);
963 int count;
964
965 if (dma_debug_disabled())
966 return 0;
967
968 switch (action) {
969 case BUS_NOTIFY_UNBOUND_DRIVER:
970 count = device_dma_allocations(dev, &entry);
971 if (count == 0)
972 break;
973 err_printk(dev, entry, "DMA-API: device driver has pending "
974 "DMA allocations while released from device "
975 "[count=%d]\n"
976 "One of leaked entries details: "
977 "[device address=0x%016llx] [size=%llu bytes] "
978 "[mapped with %s] [mapped as %s]\n",
979 count, entry->dev_addr, entry->size,
980 dir2name[entry->direction], type2name[entry->type]);
981 break;
982 default:
983 break;
984 }
985
986 return 0;
987}
988
989void dma_debug_add_bus(struct bus_type *bus)
990{
991 struct notifier_block *nb;
992
993 if (dma_debug_disabled())
994 return;
995
996 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
997 if (nb == NULL) {
998 pr_err("dma_debug_add_bus: out of memory\n");
999 return;
1000 }
1001
1002 nb->notifier_call = dma_debug_device_change;
1003
1004 bus_register_notifier(bus, nb);
1005}
1006
1007/*
1008 * Let the architectures decide how many entries should be preallocated.
1009 */
1010void dma_debug_init(u32 num_entries)
1011{
1012 int i;
1013
1014 /* Do not use dma_debug_initialized here, since we really want to be
1015 * called to set dma_debug_initialized
1016 */
1017 if (global_disable)
1018 return;
1019
1020 for (i = 0; i < HASH_SIZE; ++i) {
1021 INIT_LIST_HEAD(&dma_entry_hash[i].list);
1022 spin_lock_init(&dma_entry_hash[i].lock);
1023 }
1024
1025 if (dma_debug_fs_init() != 0) {
1026 pr_err("DMA-API: error creating debugfs entries - disabling\n");
1027 global_disable = true;
1028
1029 return;
1030 }
1031
1032 if (req_entries)
1033 num_entries = req_entries;
1034
1035 if (prealloc_memory(num_entries) != 0) {
1036 pr_err("DMA-API: debugging out of memory error - disabled\n");
1037 global_disable = true;
1038
1039 return;
1040 }
1041
1042 nr_total_entries = num_free_entries;
1043
1044 dma_debug_initialized = true;
1045
1046 pr_info("DMA-API: debugging enabled by kernel config\n");
1047}
1048
1049static __init int dma_debug_cmdline(char *str)
1050{
1051 if (!str)
1052 return -EINVAL;
1053
1054 if (strncmp(str, "off", 3) == 0) {
1055 pr_info("DMA-API: debugging disabled on kernel command line\n");
1056 global_disable = true;
1057 }
1058
1059 return 0;
1060}
1061
1062static __init int dma_debug_entries_cmdline(char *str)
1063{
1064 int res;
1065
1066 if (!str)
1067 return -EINVAL;
1068
1069 res = get_option(&str, &req_entries);
1070
1071 if (!res)
1072 req_entries = 0;
1073
1074 return 0;
1075}
1076
1077__setup("dma_debug=", dma_debug_cmdline);
1078__setup("dma_debug_entries=", dma_debug_entries_cmdline);
1079
1080static void check_unmap(struct dma_debug_entry *ref)
1081{
1082 struct dma_debug_entry *entry;
1083 struct hash_bucket *bucket;
1084 unsigned long flags;
1085
1086 bucket = get_hash_bucket(ref, &flags);
1087 entry = bucket_find_exact(bucket, ref);
1088
1089 if (!entry) {
1090 /* must drop lock before calling dma_mapping_error */
1091 put_hash_bucket(bucket, &flags);
1092
1093 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1094 err_printk(ref->dev, NULL,
1095 "DMA-API: device driver tries to free an "
1096 "invalid DMA memory address\n");
1097 } else {
1098 err_printk(ref->dev, NULL,
1099 "DMA-API: device driver tries to free DMA "
1100 "memory it has not allocated [device "
1101 "address=0x%016llx] [size=%llu bytes]\n",
1102 ref->dev_addr, ref->size);
1103 }
1104 return;
1105 }
1106
1107 if (ref->size != entry->size) {
1108 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1109 "DMA memory with different size "
1110 "[device address=0x%016llx] [map size=%llu bytes] "
1111 "[unmap size=%llu bytes]\n",
1112 ref->dev_addr, entry->size, ref->size);
1113 }
1114
1115 if (ref->type != entry->type) {
1116 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1117 "DMA memory with wrong function "
1118 "[device address=0x%016llx] [size=%llu bytes] "
1119 "[mapped as %s] [unmapped as %s]\n",
1120 ref->dev_addr, ref->size,
1121 type2name[entry->type], type2name[ref->type]);
1122 } else if ((entry->type == dma_debug_coherent) &&
1123 (phys_addr(ref) != phys_addr(entry))) {
1124 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1125 "DMA memory with different CPU address "
1126 "[device address=0x%016llx] [size=%llu bytes] "
1127 "[cpu alloc address=0x%016llx] "
1128 "[cpu free address=0x%016llx]",
1129 ref->dev_addr, ref->size,
1130 phys_addr(entry),
1131 phys_addr(ref));
1132 }
1133
1134 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1135 ref->sg_call_ents != entry->sg_call_ents) {
1136 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1137 "DMA sg list with different entry count "
1138 "[map count=%d] [unmap count=%d]\n",
1139 entry->sg_call_ents, ref->sg_call_ents);
1140 }
1141
1142 /*
1143 * This may be no bug in reality - but most implementations of the
1144 * DMA API don't handle this properly, so check for it here
1145 */
1146 if (ref->direction != entry->direction) {
1147 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1148 "DMA memory with different direction "
1149 "[device address=0x%016llx] [size=%llu bytes] "
1150 "[mapped with %s] [unmapped with %s]\n",
1151 ref->dev_addr, ref->size,
1152 dir2name[entry->direction],
1153 dir2name[ref->direction]);
1154 }
1155
1156 /*
1157 * Drivers should use dma_mapping_error() to check the returned
1158 * addresses of dma_map_single() and dma_map_page().
1159 * If not, print this warning message. See Documentation/DMA-API.txt.
1160 */
1161 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1162 err_printk(ref->dev, entry,
1163 "DMA-API: device driver failed to check map error"
1164 "[device address=0x%016llx] [size=%llu bytes] "
1165 "[mapped as %s]",
1166 ref->dev_addr, ref->size,
1167 type2name[entry->type]);
1168 }
1169
1170 hash_bucket_del(entry);
1171 dma_entry_free(entry);
1172
1173 put_hash_bucket(bucket, &flags);
1174}
1175
1176static void check_for_stack(struct device *dev,
1177 struct page *page, size_t offset)
1178{
1179 void *addr;
1180 struct vm_struct *stack_vm_area = task_stack_vm_area(current);
1181
1182 if (!stack_vm_area) {
1183 /* Stack is direct-mapped. */
1184 if (PageHighMem(page))
1185 return;
1186 addr = page_address(page) + offset;
1187 if (object_is_on_stack(addr))
1188 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [addr=%p]\n", addr);
1189 } else {
1190 /* Stack is vmalloced. */
1191 int i;
1192
1193 for (i = 0; i < stack_vm_area->nr_pages; i++) {
1194 if (page != stack_vm_area->pages[i])
1195 continue;
1196
1197 addr = (u8 *)current->stack + i * PAGE_SIZE + offset;
1198 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [probable addr=%p]\n", addr);
1199 break;
1200 }
1201 }
1202}
1203
1204static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
1205{
1206 unsigned long a1 = (unsigned long)addr;
1207 unsigned long b1 = a1 + len;
1208 unsigned long a2 = (unsigned long)start;
1209 unsigned long b2 = (unsigned long)end;
1210
1211 return !(b1 <= a2 || a1 >= b2);
1212}
1213
1214static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
1215{
1216 if (overlap(addr, len, _stext, _etext) ||
1217 overlap(addr, len, __start_rodata, __end_rodata))
1218 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1219}
1220
1221static void check_sync(struct device *dev,
1222 struct dma_debug_entry *ref,
1223 bool to_cpu)
1224{
1225 struct dma_debug_entry *entry;
1226 struct hash_bucket *bucket;
1227 unsigned long flags;
1228
1229 bucket = get_hash_bucket(ref, &flags);
1230
1231 entry = bucket_find_contain(&bucket, ref, &flags);
1232
1233 if (!entry) {
1234 err_printk(dev, NULL, "DMA-API: device driver tries "
1235 "to sync DMA memory it has not allocated "
1236 "[device address=0x%016llx] [size=%llu bytes]\n",
1237 (unsigned long long)ref->dev_addr, ref->size);
1238 goto out;
1239 }
1240
1241 if (ref->size > entry->size) {
1242 err_printk(dev, entry, "DMA-API: device driver syncs"
1243 " DMA memory outside allocated range "
1244 "[device address=0x%016llx] "
1245 "[allocation size=%llu bytes] "
1246 "[sync offset+size=%llu]\n",
1247 entry->dev_addr, entry->size,
1248 ref->size);
1249 }
1250
1251 if (entry->direction == DMA_BIDIRECTIONAL)
1252 goto out;
1253
1254 if (ref->direction != entry->direction) {
1255 err_printk(dev, entry, "DMA-API: device driver syncs "
1256 "DMA memory with different direction "
1257 "[device address=0x%016llx] [size=%llu bytes] "
1258 "[mapped with %s] [synced with %s]\n",
1259 (unsigned long long)ref->dev_addr, entry->size,
1260 dir2name[entry->direction],
1261 dir2name[ref->direction]);
1262 }
1263
1264 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
1265 !(ref->direction == DMA_TO_DEVICE))
1266 err_printk(dev, entry, "DMA-API: device driver syncs "
1267 "device read-only DMA memory for cpu "
1268 "[device address=0x%016llx] [size=%llu bytes] "
1269 "[mapped with %s] [synced with %s]\n",
1270 (unsigned long long)ref->dev_addr, entry->size,
1271 dir2name[entry->direction],
1272 dir2name[ref->direction]);
1273
1274 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
1275 !(ref->direction == DMA_FROM_DEVICE))
1276 err_printk(dev, entry, "DMA-API: device driver syncs "
1277 "device write-only DMA memory to device "
1278 "[device address=0x%016llx] [size=%llu bytes] "
1279 "[mapped with %s] [synced with %s]\n",
1280 (unsigned long long)ref->dev_addr, entry->size,
1281 dir2name[entry->direction],
1282 dir2name[ref->direction]);
1283
1284 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1285 ref->sg_call_ents != entry->sg_call_ents) {
1286 err_printk(ref->dev, entry, "DMA-API: device driver syncs "
1287 "DMA sg list with different entry count "
1288 "[map count=%d] [sync count=%d]\n",
1289 entry->sg_call_ents, ref->sg_call_ents);
1290 }
1291
1292out:
1293 put_hash_bucket(bucket, &flags);
1294}
1295
1296void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1297 size_t size, int direction, dma_addr_t dma_addr,
1298 bool map_single)
1299{
1300 struct dma_debug_entry *entry;
1301
1302 if (unlikely(dma_debug_disabled()))
1303 return;
1304
1305 if (dma_mapping_error(dev, dma_addr))
1306 return;
1307
1308 entry = dma_entry_alloc();
1309 if (!entry)
1310 return;
1311
1312 entry->dev = dev;
1313 entry->type = dma_debug_page;
1314 entry->pfn = page_to_pfn(page);
1315 entry->offset = offset,
1316 entry->dev_addr = dma_addr;
1317 entry->size = size;
1318 entry->direction = direction;
1319 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1320
1321 if (map_single)
1322 entry->type = dma_debug_single;
1323
1324 check_for_stack(dev, page, offset);
1325
1326 if (!PageHighMem(page)) {
1327 void *addr = page_address(page) + offset;
1328
1329 check_for_illegal_area(dev, addr, size);
1330 }
1331
1332 add_dma_entry(entry);
1333}
1334EXPORT_SYMBOL(debug_dma_map_page);
1335
1336void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1337{
1338 struct dma_debug_entry ref;
1339 struct dma_debug_entry *entry;
1340 struct hash_bucket *bucket;
1341 unsigned long flags;
1342
1343 if (unlikely(dma_debug_disabled()))
1344 return;
1345
1346 ref.dev = dev;
1347 ref.dev_addr = dma_addr;
1348 bucket = get_hash_bucket(&ref, &flags);
1349
1350 list_for_each_entry(entry, &bucket->list, list) {
1351 if (!exact_match(&ref, entry))
1352 continue;
1353
1354 /*
1355 * The same physical address can be mapped multiple
1356 * times. Without a hardware IOMMU this results in the
1357 * same device addresses being put into the dma-debug
1358 * hash multiple times too. This can result in false
1359 * positives being reported. Therefore we implement a
1360 * best-fit algorithm here which updates the first entry
1361 * from the hash which fits the reference value and is
1362 * not currently listed as being checked.
1363 */
1364 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1365 entry->map_err_type = MAP_ERR_CHECKED;
1366 break;
1367 }
1368 }
1369
1370 put_hash_bucket(bucket, &flags);
1371}
1372EXPORT_SYMBOL(debug_dma_mapping_error);
1373
1374void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1375 size_t size, int direction, bool map_single)
1376{
1377 struct dma_debug_entry ref = {
1378 .type = dma_debug_page,
1379 .dev = dev,
1380 .dev_addr = addr,
1381 .size = size,
1382 .direction = direction,
1383 };
1384
1385 if (unlikely(dma_debug_disabled()))
1386 return;
1387
1388 if (map_single)
1389 ref.type = dma_debug_single;
1390
1391 check_unmap(&ref);
1392}
1393EXPORT_SYMBOL(debug_dma_unmap_page);
1394
1395void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1396 int nents, int mapped_ents, int direction)
1397{
1398 struct dma_debug_entry *entry;
1399 struct scatterlist *s;
1400 int i;
1401
1402 if (unlikely(dma_debug_disabled()))
1403 return;
1404
1405 for_each_sg(sg, s, mapped_ents, i) {
1406 entry = dma_entry_alloc();
1407 if (!entry)
1408 return;
1409
1410 entry->type = dma_debug_sg;
1411 entry->dev = dev;
1412 entry->pfn = page_to_pfn(sg_page(s));
1413 entry->offset = s->offset,
1414 entry->size = sg_dma_len(s);
1415 entry->dev_addr = sg_dma_address(s);
1416 entry->direction = direction;
1417 entry->sg_call_ents = nents;
1418 entry->sg_mapped_ents = mapped_ents;
1419
1420 check_for_stack(dev, sg_page(s), s->offset);
1421
1422 if (!PageHighMem(sg_page(s))) {
1423 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1424 }
1425
1426 add_dma_entry(entry);
1427 }
1428}
1429EXPORT_SYMBOL(debug_dma_map_sg);
1430
1431static int get_nr_mapped_entries(struct device *dev,
1432 struct dma_debug_entry *ref)
1433{
1434 struct dma_debug_entry *entry;
1435 struct hash_bucket *bucket;
1436 unsigned long flags;
1437 int mapped_ents;
1438
1439 bucket = get_hash_bucket(ref, &flags);
1440 entry = bucket_find_exact(bucket, ref);
1441 mapped_ents = 0;
1442
1443 if (entry)
1444 mapped_ents = entry->sg_mapped_ents;
1445 put_hash_bucket(bucket, &flags);
1446
1447 return mapped_ents;
1448}
1449
1450void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1451 int nelems, int dir)
1452{
1453 struct scatterlist *s;
1454 int mapped_ents = 0, i;
1455
1456 if (unlikely(dma_debug_disabled()))
1457 return;
1458
1459 for_each_sg(sglist, s, nelems, i) {
1460
1461 struct dma_debug_entry ref = {
1462 .type = dma_debug_sg,
1463 .dev = dev,
1464 .pfn = page_to_pfn(sg_page(s)),
1465 .offset = s->offset,
1466 .dev_addr = sg_dma_address(s),
1467 .size = sg_dma_len(s),
1468 .direction = dir,
1469 .sg_call_ents = nelems,
1470 };
1471
1472 if (mapped_ents && i >= mapped_ents)
1473 break;
1474
1475 if (!i)
1476 mapped_ents = get_nr_mapped_entries(dev, &ref);
1477
1478 check_unmap(&ref);
1479 }
1480}
1481EXPORT_SYMBOL(debug_dma_unmap_sg);
1482
1483void debug_dma_alloc_coherent(struct device *dev, size_t size,
1484 dma_addr_t dma_addr, void *virt)
1485{
1486 struct dma_debug_entry *entry;
1487
1488 if (unlikely(dma_debug_disabled()))
1489 return;
1490
1491 if (unlikely(virt == NULL))
1492 return;
1493
1494 /* handle vmalloc and linear addresses */
1495 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1496 return;
1497
1498 entry = dma_entry_alloc();
1499 if (!entry)
1500 return;
1501
1502 entry->type = dma_debug_coherent;
1503 entry->dev = dev;
1504 entry->offset = offset_in_page(virt);
1505 entry->size = size;
1506 entry->dev_addr = dma_addr;
1507 entry->direction = DMA_BIDIRECTIONAL;
1508
1509 if (is_vmalloc_addr(virt))
1510 entry->pfn = vmalloc_to_pfn(virt);
1511 else
1512 entry->pfn = page_to_pfn(virt_to_page(virt));
1513
1514 add_dma_entry(entry);
1515}
1516EXPORT_SYMBOL(debug_dma_alloc_coherent);
1517
1518void debug_dma_free_coherent(struct device *dev, size_t size,
1519 void *virt, dma_addr_t addr)
1520{
1521 struct dma_debug_entry ref = {
1522 .type = dma_debug_coherent,
1523 .dev = dev,
1524 .offset = offset_in_page(virt),
1525 .dev_addr = addr,
1526 .size = size,
1527 .direction = DMA_BIDIRECTIONAL,
1528 };
1529
1530 /* handle vmalloc and linear addresses */
1531 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1532 return;
1533
1534 if (is_vmalloc_addr(virt))
1535 ref.pfn = vmalloc_to_pfn(virt);
1536 else
1537 ref.pfn = page_to_pfn(virt_to_page(virt));
1538
1539 if (unlikely(dma_debug_disabled()))
1540 return;
1541
1542 check_unmap(&ref);
1543}
1544EXPORT_SYMBOL(debug_dma_free_coherent);
1545
1546void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size,
1547 int direction, dma_addr_t dma_addr)
1548{
1549 struct dma_debug_entry *entry;
1550
1551 if (unlikely(dma_debug_disabled()))
1552 return;
1553
1554 entry = dma_entry_alloc();
1555 if (!entry)
1556 return;
1557
1558 entry->type = dma_debug_resource;
1559 entry->dev = dev;
1560 entry->pfn = PHYS_PFN(addr);
1561 entry->offset = offset_in_page(addr);
1562 entry->size = size;
1563 entry->dev_addr = dma_addr;
1564 entry->direction = direction;
1565 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1566
1567 add_dma_entry(entry);
1568}
1569EXPORT_SYMBOL(debug_dma_map_resource);
1570
1571void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr,
1572 size_t size, int direction)
1573{
1574 struct dma_debug_entry ref = {
1575 .type = dma_debug_resource,
1576 .dev = dev,
1577 .dev_addr = dma_addr,
1578 .size = size,
1579 .direction = direction,
1580 };
1581
1582 if (unlikely(dma_debug_disabled()))
1583 return;
1584
1585 check_unmap(&ref);
1586}
1587EXPORT_SYMBOL(debug_dma_unmap_resource);
1588
1589void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1590 size_t size, int direction)
1591{
1592 struct dma_debug_entry ref;
1593
1594 if (unlikely(dma_debug_disabled()))
1595 return;
1596
1597 ref.type = dma_debug_single;
1598 ref.dev = dev;
1599 ref.dev_addr = dma_handle;
1600 ref.size = size;
1601 ref.direction = direction;
1602 ref.sg_call_ents = 0;
1603
1604 check_sync(dev, &ref, true);
1605}
1606EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1607
1608void debug_dma_sync_single_for_device(struct device *dev,
1609 dma_addr_t dma_handle, size_t size,
1610 int direction)
1611{
1612 struct dma_debug_entry ref;
1613
1614 if (unlikely(dma_debug_disabled()))
1615 return;
1616
1617 ref.type = dma_debug_single;
1618 ref.dev = dev;
1619 ref.dev_addr = dma_handle;
1620 ref.size = size;
1621 ref.direction = direction;
1622 ref.sg_call_ents = 0;
1623
1624 check_sync(dev, &ref, false);
1625}
1626EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1627
1628void debug_dma_sync_single_range_for_cpu(struct device *dev,
1629 dma_addr_t dma_handle,
1630 unsigned long offset, size_t size,
1631 int direction)
1632{
1633 struct dma_debug_entry ref;
1634
1635 if (unlikely(dma_debug_disabled()))
1636 return;
1637
1638 ref.type = dma_debug_single;
1639 ref.dev = dev;
1640 ref.dev_addr = dma_handle;
1641 ref.size = offset + size;
1642 ref.direction = direction;
1643 ref.sg_call_ents = 0;
1644
1645 check_sync(dev, &ref, true);
1646}
1647EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1648
1649void debug_dma_sync_single_range_for_device(struct device *dev,
1650 dma_addr_t dma_handle,
1651 unsigned long offset,
1652 size_t size, int direction)
1653{
1654 struct dma_debug_entry ref;
1655
1656 if (unlikely(dma_debug_disabled()))
1657 return;
1658
1659 ref.type = dma_debug_single;
1660 ref.dev = dev;
1661 ref.dev_addr = dma_handle;
1662 ref.size = offset + size;
1663 ref.direction = direction;
1664 ref.sg_call_ents = 0;
1665
1666 check_sync(dev, &ref, false);
1667}
1668EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1669
1670void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1671 int nelems, int direction)
1672{
1673 struct scatterlist *s;
1674 int mapped_ents = 0, i;
1675
1676 if (unlikely(dma_debug_disabled()))
1677 return;
1678
1679 for_each_sg(sg, s, nelems, i) {
1680
1681 struct dma_debug_entry ref = {
1682 .type = dma_debug_sg,
1683 .dev = dev,
1684 .pfn = page_to_pfn(sg_page(s)),
1685 .offset = s->offset,
1686 .dev_addr = sg_dma_address(s),
1687 .size = sg_dma_len(s),
1688 .direction = direction,
1689 .sg_call_ents = nelems,
1690 };
1691
1692 if (!i)
1693 mapped_ents = get_nr_mapped_entries(dev, &ref);
1694
1695 if (i >= mapped_ents)
1696 break;
1697
1698 check_sync(dev, &ref, true);
1699 }
1700}
1701EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1702
1703void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1704 int nelems, int direction)
1705{
1706 struct scatterlist *s;
1707 int mapped_ents = 0, i;
1708
1709 if (unlikely(dma_debug_disabled()))
1710 return;
1711
1712 for_each_sg(sg, s, nelems, i) {
1713
1714 struct dma_debug_entry ref = {
1715 .type = dma_debug_sg,
1716 .dev = dev,
1717 .pfn = page_to_pfn(sg_page(s)),
1718 .offset = s->offset,
1719 .dev_addr = sg_dma_address(s),
1720 .size = sg_dma_len(s),
1721 .direction = direction,
1722 .sg_call_ents = nelems,
1723 };
1724 if (!i)
1725 mapped_ents = get_nr_mapped_entries(dev, &ref);
1726
1727 if (i >= mapped_ents)
1728 break;
1729
1730 check_sync(dev, &ref, false);
1731 }
1732}
1733EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1734
1735static int __init dma_debug_driver_setup(char *str)
1736{
1737 int i;
1738
1739 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1740 current_driver_name[i] = *str;
1741 if (*str == 0)
1742 break;
1743 }
1744
1745 if (current_driver_name[0])
1746 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1747 current_driver_name);
1748
1749
1750 return 1;
1751}
1752__setup("dma_debug_driver=", dma_debug_driver_setup);
diff --git a/lib/dma-direct.c b/lib/dma-direct.c
deleted file mode 100644
index bbfb229aa067..000000000000
--- a/lib/dma-direct.c
+++ /dev/null
@@ -1,185 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * DMA operations that map physical memory directly without using an IOMMU or
4 * flushing caches.
5 */
6#include <linux/export.h>
7#include <linux/mm.h>
8#include <linux/dma-direct.h>
9#include <linux/scatterlist.h>
10#include <linux/dma-contiguous.h>
11#include <linux/pfn.h>
12#include <linux/set_memory.h>
13
14#define DIRECT_MAPPING_ERROR 0
15
16/*
17 * Most architectures use ZONE_DMA for the first 16 Megabytes, but
18 * some use it for entirely different regions:
19 */
20#ifndef ARCH_ZONE_DMA_BITS
21#define ARCH_ZONE_DMA_BITS 24
22#endif
23
24/*
25 * For AMD SEV all DMA must be to unencrypted addresses.
26 */
27static inline bool force_dma_unencrypted(void)
28{
29 return sev_active();
30}
31
32static bool
33check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
34 const char *caller)
35{
36 if (unlikely(dev && !dma_capable(dev, dma_addr, size))) {
37 if (*dev->dma_mask >= DMA_BIT_MASK(32)) {
38 dev_err(dev,
39 "%s: overflow %pad+%zu of device mask %llx\n",
40 caller, &dma_addr, size, *dev->dma_mask);
41 }
42 return false;
43 }
44 return true;
45}
46
47static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
48{
49 dma_addr_t addr = force_dma_unencrypted() ?
50 __phys_to_dma(dev, phys) : phys_to_dma(dev, phys);
51 return addr + size - 1 <= dev->coherent_dma_mask;
52}
53
54void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
55 gfp_t gfp, unsigned long attrs)
56{
57 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
58 int page_order = get_order(size);
59 struct page *page = NULL;
60 void *ret;
61
62 /* we always manually zero the memory once we are done: */
63 gfp &= ~__GFP_ZERO;
64
65 /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */
66 if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
67 gfp |= GFP_DMA;
68 if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA))
69 gfp |= GFP_DMA32;
70
71again:
72 /* CMA can be used only in the context which permits sleeping */
73 if (gfpflags_allow_blocking(gfp)) {
74 page = dma_alloc_from_contiguous(dev, count, page_order, gfp);
75 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
76 dma_release_from_contiguous(dev, page, count);
77 page = NULL;
78 }
79 }
80 if (!page)
81 page = alloc_pages_node(dev_to_node(dev), gfp, page_order);
82
83 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
84 __free_pages(page, page_order);
85 page = NULL;
86
87 if (IS_ENABLED(CONFIG_ZONE_DMA) &&
88 dev->coherent_dma_mask < DMA_BIT_MASK(32) &&
89 !(gfp & GFP_DMA)) {
90 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
91 goto again;
92 }
93 }
94
95 if (!page)
96 return NULL;
97 ret = page_address(page);
98 if (force_dma_unencrypted()) {
99 set_memory_decrypted((unsigned long)ret, 1 << page_order);
100 *dma_handle = __phys_to_dma(dev, page_to_phys(page));
101 } else {
102 *dma_handle = phys_to_dma(dev, page_to_phys(page));
103 }
104 memset(ret, 0, size);
105 return ret;
106}
107
108/*
109 * NOTE: this function must never look at the dma_addr argument, because we want
110 * to be able to use it as a helper for iommu implementations as well.
111 */
112void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
113 dma_addr_t dma_addr, unsigned long attrs)
114{
115 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
116 unsigned int page_order = get_order(size);
117
118 if (force_dma_unencrypted())
119 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
120 if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count))
121 free_pages((unsigned long)cpu_addr, page_order);
122}
123
124static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
125 unsigned long offset, size_t size, enum dma_data_direction dir,
126 unsigned long attrs)
127{
128 dma_addr_t dma_addr = phys_to_dma(dev, page_to_phys(page)) + offset;
129
130 if (!check_addr(dev, dma_addr, size, __func__))
131 return DIRECT_MAPPING_ERROR;
132 return dma_addr;
133}
134
135static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
136 int nents, enum dma_data_direction dir, unsigned long attrs)
137{
138 int i;
139 struct scatterlist *sg;
140
141 for_each_sg(sgl, sg, nents, i) {
142 BUG_ON(!sg_page(sg));
143
144 sg_dma_address(sg) = phys_to_dma(dev, sg_phys(sg));
145 if (!check_addr(dev, sg_dma_address(sg), sg->length, __func__))
146 return 0;
147 sg_dma_len(sg) = sg->length;
148 }
149
150 return nents;
151}
152
153int dma_direct_supported(struct device *dev, u64 mask)
154{
155#ifdef CONFIG_ZONE_DMA
156 if (mask < DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
157 return 0;
158#else
159 /*
160 * Because 32-bit DMA masks are so common we expect every architecture
161 * to be able to satisfy them - either by not supporting more physical
162 * memory, or by providing a ZONE_DMA32. If neither is the case, the
163 * architecture needs to use an IOMMU instead of the direct mapping.
164 */
165 if (mask < DMA_BIT_MASK(32))
166 return 0;
167#endif
168 return 1;
169}
170
171static int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr)
172{
173 return dma_addr == DIRECT_MAPPING_ERROR;
174}
175
176const struct dma_map_ops dma_direct_ops = {
177 .alloc = dma_direct_alloc,
178 .free = dma_direct_free,
179 .map_page = dma_direct_map_page,
180 .map_sg = dma_direct_map_sg,
181 .dma_supported = dma_direct_supported,
182 .mapping_error = dma_direct_mapping_error,
183 .is_phys = 1,
184};
185EXPORT_SYMBOL(dma_direct_ops);
diff --git a/lib/dma-virt.c b/lib/dma-virt.c
deleted file mode 100644
index 8e61a02ef9ca..000000000000
--- a/lib/dma-virt.c
+++ /dev/null
@@ -1,61 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * lib/dma-virt.c
4 *
5 * DMA operations that map to virtual addresses without flushing memory.
6 */
7#include <linux/export.h>
8#include <linux/mm.h>
9#include <linux/dma-mapping.h>
10#include <linux/scatterlist.h>
11
12static void *dma_virt_alloc(struct device *dev, size_t size,
13 dma_addr_t *dma_handle, gfp_t gfp,
14 unsigned long attrs)
15{
16 void *ret;
17
18 ret = (void *)__get_free_pages(gfp, get_order(size));
19 if (ret)
20 *dma_handle = (uintptr_t)ret;
21 return ret;
22}
23
24static void dma_virt_free(struct device *dev, size_t size,
25 void *cpu_addr, dma_addr_t dma_addr,
26 unsigned long attrs)
27{
28 free_pages((unsigned long)cpu_addr, get_order(size));
29}
30
31static dma_addr_t dma_virt_map_page(struct device *dev, struct page *page,
32 unsigned long offset, size_t size,
33 enum dma_data_direction dir,
34 unsigned long attrs)
35{
36 return (uintptr_t)(page_address(page) + offset);
37}
38
39static int dma_virt_map_sg(struct device *dev, struct scatterlist *sgl,
40 int nents, enum dma_data_direction dir,
41 unsigned long attrs)
42{
43 int i;
44 struct scatterlist *sg;
45
46 for_each_sg(sgl, sg, nents, i) {
47 BUG_ON(!sg_page(sg));
48 sg_dma_address(sg) = (uintptr_t)sg_virt(sg);
49 sg_dma_len(sg) = sg->length;
50 }
51
52 return nents;
53}
54
55const struct dma_map_ops dma_virt_ops = {
56 .alloc = dma_virt_alloc,
57 .free = dma_virt_free,
58 .map_page = dma_virt_map_page,
59 .map_sg = dma_virt_map_sg,
60};
61EXPORT_SYMBOL(dma_virt_ops);
diff --git a/lib/idr.c b/lib/idr.c
index 823b813f08f8..ed9c169c12bd 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -4,9 +4,9 @@
4#include <linux/idr.h> 4#include <linux/idr.h>
5#include <linux/slab.h> 5#include <linux/slab.h>
6#include <linux/spinlock.h> 6#include <linux/spinlock.h>
7#include <linux/xarray.h>
7 8
8DEFINE_PER_CPU(struct ida_bitmap *, ida_bitmap); 9DEFINE_PER_CPU(struct ida_bitmap *, ida_bitmap);
9static DEFINE_SPINLOCK(simple_ida_lock);
10 10
11/** 11/**
12 * idr_alloc_u32() - Allocate an ID. 12 * idr_alloc_u32() - Allocate an ID.
@@ -581,7 +581,7 @@ again:
581 if (!ida_pre_get(ida, gfp_mask)) 581 if (!ida_pre_get(ida, gfp_mask))
582 return -ENOMEM; 582 return -ENOMEM;
583 583
584 spin_lock_irqsave(&simple_ida_lock, flags); 584 xa_lock_irqsave(&ida->ida_rt, flags);
585 ret = ida_get_new_above(ida, start, &id); 585 ret = ida_get_new_above(ida, start, &id);
586 if (!ret) { 586 if (!ret) {
587 if (id > max) { 587 if (id > max) {
@@ -591,7 +591,7 @@ again:
591 ret = id; 591 ret = id;
592 } 592 }
593 } 593 }
594 spin_unlock_irqrestore(&simple_ida_lock, flags); 594 xa_unlock_irqrestore(&ida->ida_rt, flags);
595 595
596 if (unlikely(ret == -EAGAIN)) 596 if (unlikely(ret == -EAGAIN))
597 goto again; 597 goto again;
@@ -615,8 +615,8 @@ void ida_simple_remove(struct ida *ida, unsigned int id)
615 unsigned long flags; 615 unsigned long flags;
616 616
617 BUG_ON((int)id < 0); 617 BUG_ON((int)id < 0);
618 spin_lock_irqsave(&simple_ida_lock, flags); 618 xa_lock_irqsave(&ida->ida_rt, flags);
619 ida_remove(ida, id); 619 ida_remove(ida, id);
620 spin_unlock_irqrestore(&simple_ida_lock, flags); 620 xa_unlock_irqrestore(&ida->ida_rt, flags);
621} 621}
622EXPORT_SYMBOL(ida_simple_remove); 622EXPORT_SYMBOL(ida_simple_remove);
diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c
index 835242e74aaa..75509a1511a3 100644
--- a/lib/interval_tree_test.c
+++ b/lib/interval_tree_test.c
@@ -64,11 +64,12 @@ static int interval_tree_test_init(void)
64 unsigned long results; 64 unsigned long results;
65 cycles_t time1, time2, time; 65 cycles_t time1, time2, time;
66 66
67 nodes = kmalloc(nnodes * sizeof(struct interval_tree_node), GFP_KERNEL); 67 nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
68 GFP_KERNEL);
68 if (!nodes) 69 if (!nodes)
69 return -ENOMEM; 70 return -ENOMEM;
70 71
71 queries = kmalloc(nsearches * sizeof(int), GFP_KERNEL); 72 queries = kmalloc_array(nsearches, sizeof(int), GFP_KERNEL);
72 if (!queries) { 73 if (!queries) {
73 kfree(nodes); 74 kfree(nodes);
74 return -ENOMEM; 75 return -ENOMEM;
diff --git a/lib/iommu-common.c b/lib/iommu-common.c
deleted file mode 100644
index 55b00de106b5..000000000000
--- a/lib/iommu-common.c
+++ /dev/null
@@ -1,267 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * IOMMU mmap management and range allocation functions.
4 * Based almost entirely upon the powerpc iommu allocator.
5 */
6
7#include <linux/export.h>
8#include <linux/bitmap.h>
9#include <linux/bug.h>
10#include <linux/iommu-helper.h>
11#include <linux/iommu-common.h>
12#include <linux/dma-mapping.h>
13#include <linux/hash.h>
14
15static unsigned long iommu_large_alloc = 15;
16
17static DEFINE_PER_CPU(unsigned int, iommu_hash_common);
18
19static inline bool need_flush(struct iommu_map_table *iommu)
20{
21 return ((iommu->flags & IOMMU_NEED_FLUSH) != 0);
22}
23
24static inline void set_flush(struct iommu_map_table *iommu)
25{
26 iommu->flags |= IOMMU_NEED_FLUSH;
27}
28
29static inline void clear_flush(struct iommu_map_table *iommu)
30{
31 iommu->flags &= ~IOMMU_NEED_FLUSH;
32}
33
34static void setup_iommu_pool_hash(void)
35{
36 unsigned int i;
37 static bool do_once;
38
39 if (do_once)
40 return;
41 do_once = true;
42 for_each_possible_cpu(i)
43 per_cpu(iommu_hash_common, i) = hash_32(i, IOMMU_POOL_HASHBITS);
44}
45
46/*
47 * Initialize iommu_pool entries for the iommu_map_table. `num_entries'
48 * is the number of table entries. If `large_pool' is set to true,
49 * the top 1/4 of the table will be set aside for pool allocations
50 * of more than iommu_large_alloc pages.
51 */
52void iommu_tbl_pool_init(struct iommu_map_table *iommu,
53 unsigned long num_entries,
54 u32 table_shift,
55 void (*lazy_flush)(struct iommu_map_table *),
56 bool large_pool, u32 npools,
57 bool skip_span_boundary_check)
58{
59 unsigned int start, i;
60 struct iommu_pool *p = &(iommu->large_pool);
61
62 setup_iommu_pool_hash();
63 if (npools == 0)
64 iommu->nr_pools = IOMMU_NR_POOLS;
65 else
66 iommu->nr_pools = npools;
67 BUG_ON(npools > IOMMU_NR_POOLS);
68
69 iommu->table_shift = table_shift;
70 iommu->lazy_flush = lazy_flush;
71 start = 0;
72 if (skip_span_boundary_check)
73 iommu->flags |= IOMMU_NO_SPAN_BOUND;
74 if (large_pool)
75 iommu->flags |= IOMMU_HAS_LARGE_POOL;
76
77 if (!large_pool)
78 iommu->poolsize = num_entries/iommu->nr_pools;
79 else
80 iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools;
81 for (i = 0; i < iommu->nr_pools; i++) {
82 spin_lock_init(&(iommu->pools[i].lock));
83 iommu->pools[i].start = start;
84 iommu->pools[i].hint = start;
85 start += iommu->poolsize; /* start for next pool */
86 iommu->pools[i].end = start - 1;
87 }
88 if (!large_pool)
89 return;
90 /* initialize large_pool */
91 spin_lock_init(&(p->lock));
92 p->start = start;
93 p->hint = p->start;
94 p->end = num_entries;
95}
96EXPORT_SYMBOL(iommu_tbl_pool_init);
97
98unsigned long iommu_tbl_range_alloc(struct device *dev,
99 struct iommu_map_table *iommu,
100 unsigned long npages,
101 unsigned long *handle,
102 unsigned long mask,
103 unsigned int align_order)
104{
105 unsigned int pool_hash = __this_cpu_read(iommu_hash_common);
106 unsigned long n, end, start, limit, boundary_size;
107 struct iommu_pool *pool;
108 int pass = 0;
109 unsigned int pool_nr;
110 unsigned int npools = iommu->nr_pools;
111 unsigned long flags;
112 bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0);
113 bool largealloc = (large_pool && npages > iommu_large_alloc);
114 unsigned long shift;
115 unsigned long align_mask = 0;
116
117 if (align_order > 0)
118 align_mask = ~0ul >> (BITS_PER_LONG - align_order);
119
120 /* Sanity check */
121 if (unlikely(npages == 0)) {
122 WARN_ON_ONCE(1);
123 return IOMMU_ERROR_CODE;
124 }
125
126 if (largealloc) {
127 pool = &(iommu->large_pool);
128 pool_nr = 0; /* to keep compiler happy */
129 } else {
130 /* pick out pool_nr */
131 pool_nr = pool_hash & (npools - 1);
132 pool = &(iommu->pools[pool_nr]);
133 }
134 spin_lock_irqsave(&pool->lock, flags);
135
136 again:
137 if (pass == 0 && handle && *handle &&
138 (*handle >= pool->start) && (*handle < pool->end))
139 start = *handle;
140 else
141 start = pool->hint;
142
143 limit = pool->end;
144
145 /* The case below can happen if we have a small segment appended
146 * to a large, or when the previous alloc was at the very end of
147 * the available space. If so, go back to the beginning. If a
148 * flush is needed, it will get done based on the return value
149 * from iommu_area_alloc() below.
150 */
151 if (start >= limit)
152 start = pool->start;
153 shift = iommu->table_map_base >> iommu->table_shift;
154 if (limit + shift > mask) {
155 limit = mask - shift + 1;
156 /* If we're constrained on address range, first try
157 * at the masked hint to avoid O(n) search complexity,
158 * but on second pass, start at 0 in pool 0.
159 */
160 if ((start & mask) >= limit || pass > 0) {
161 spin_unlock(&(pool->lock));
162 pool = &(iommu->pools[0]);
163 spin_lock(&(pool->lock));
164 start = pool->start;
165 } else {
166 start &= mask;
167 }
168 }
169
170 if (dev)
171 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
172 1 << iommu->table_shift);
173 else
174 boundary_size = ALIGN(1ULL << 32, 1 << iommu->table_shift);
175
176 boundary_size = boundary_size >> iommu->table_shift;
177 /*
178 * if the skip_span_boundary_check had been set during init, we set
179 * things up so that iommu_is_span_boundary() merely checks if the
180 * (index + npages) < num_tsb_entries
181 */
182 if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) {
183 shift = 0;
184 boundary_size = iommu->poolsize * iommu->nr_pools;
185 }
186 n = iommu_area_alloc(iommu->map, limit, start, npages, shift,
187 boundary_size, align_mask);
188 if (n == -1) {
189 if (likely(pass == 0)) {
190 /* First failure, rescan from the beginning. */
191 pool->hint = pool->start;
192 set_flush(iommu);
193 pass++;
194 goto again;
195 } else if (!largealloc && pass <= iommu->nr_pools) {
196 spin_unlock(&(pool->lock));
197 pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
198 pool = &(iommu->pools[pool_nr]);
199 spin_lock(&(pool->lock));
200 pool->hint = pool->start;
201 set_flush(iommu);
202 pass++;
203 goto again;
204 } else {
205 /* give up */
206 n = IOMMU_ERROR_CODE;
207 goto bail;
208 }
209 }
210 if (iommu->lazy_flush &&
211 (n < pool->hint || need_flush(iommu))) {
212 clear_flush(iommu);
213 iommu->lazy_flush(iommu);
214 }
215
216 end = n + npages;
217 pool->hint = end;
218
219 /* Update handle for SG allocations */
220 if (handle)
221 *handle = end;
222bail:
223 spin_unlock_irqrestore(&(pool->lock), flags);
224
225 return n;
226}
227EXPORT_SYMBOL(iommu_tbl_range_alloc);
228
229static struct iommu_pool *get_pool(struct iommu_map_table *tbl,
230 unsigned long entry)
231{
232 struct iommu_pool *p;
233 unsigned long largepool_start = tbl->large_pool.start;
234 bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0);
235
236 /* The large pool is the last pool at the top of the table */
237 if (large_pool && entry >= largepool_start) {
238 p = &tbl->large_pool;
239 } else {
240 unsigned int pool_nr = entry / tbl->poolsize;
241
242 BUG_ON(pool_nr >= tbl->nr_pools);
243 p = &tbl->pools[pool_nr];
244 }
245 return p;
246}
247
248/* Caller supplies the index of the entry into the iommu map table
249 * itself when the mapping from dma_addr to the entry is not the
250 * default addr->entry mapping below.
251 */
252void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr,
253 unsigned long npages, unsigned long entry)
254{
255 struct iommu_pool *pool;
256 unsigned long flags;
257 unsigned long shift = iommu->table_shift;
258
259 if (entry == IOMMU_ERROR_CODE) /* use default addr->entry mapping */
260 entry = (dma_addr - iommu->table_map_base) >> shift;
261 pool = get_pool(iommu, entry);
262
263 spin_lock_irqsave(&(pool->lock), flags);
264 bitmap_clear(iommu->map, entry, npages);
265 spin_unlock_irqrestore(&(pool->lock), flags);
266}
267EXPORT_SYMBOL(iommu_tbl_range_free);
diff --git a/lib/iommu-helper.c b/lib/iommu-helper.c
index 23633c0fda4a..92a9f243c0e2 100644
--- a/lib/iommu-helper.c
+++ b/lib/iommu-helper.c
@@ -3,19 +3,8 @@
3 * IOMMU helper functions for the free area management 3 * IOMMU helper functions for the free area management
4 */ 4 */
5 5
6#include <linux/export.h>
7#include <linux/bitmap.h> 6#include <linux/bitmap.h>
8#include <linux/bug.h> 7#include <linux/iommu-helper.h>
9
10int iommu_is_span_boundary(unsigned int index, unsigned int nr,
11 unsigned long shift,
12 unsigned long boundary_size)
13{
14 BUG_ON(!is_power_of_2(boundary_size));
15
16 shift = (shift + index) & (boundary_size - 1);
17 return shift + nr > boundary_size;
18}
19 8
20unsigned long iommu_area_alloc(unsigned long *map, unsigned long size, 9unsigned long iommu_area_alloc(unsigned long *map, unsigned long size,
21 unsigned long start, unsigned int nr, 10 unsigned long start, unsigned int nr,
@@ -38,4 +27,3 @@ again:
38 } 27 }
39 return -1; 28 return -1;
40} 29}
41EXPORT_SYMBOL(iommu_area_alloc);
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 970212670b6a..7e43cd54c84c 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -573,6 +573,67 @@ size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
573} 573}
574EXPORT_SYMBOL(_copy_to_iter); 574EXPORT_SYMBOL(_copy_to_iter);
575 575
576#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
577static int copyout_mcsafe(void __user *to, const void *from, size_t n)
578{
579 if (access_ok(VERIFY_WRITE, to, n)) {
580 kasan_check_read(from, n);
581 n = copy_to_user_mcsafe((__force void *) to, from, n);
582 }
583 return n;
584}
585
586static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset,
587 const char *from, size_t len)
588{
589 unsigned long ret;
590 char *to;
591
592 to = kmap_atomic(page);
593 ret = memcpy_mcsafe(to + offset, from, len);
594 kunmap_atomic(to);
595
596 return ret;
597}
598
599size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)
600{
601 const char *from = addr;
602 unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
603
604 if (unlikely(i->type & ITER_PIPE)) {
605 WARN_ON(1);
606 return 0;
607 }
608 if (iter_is_iovec(i))
609 might_fault();
610 iterate_and_advance(i, bytes, v,
611 copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
612 ({
613 rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset,
614 (from += v.bv_len) - v.bv_len, v.bv_len);
615 if (rem) {
616 curr_addr = (unsigned long) from;
617 bytes = curr_addr - s_addr - rem;
618 return bytes;
619 }
620 }),
621 ({
622 rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len,
623 v.iov_len);
624 if (rem) {
625 curr_addr = (unsigned long) from;
626 bytes = curr_addr - s_addr - rem;
627 return bytes;
628 }
629 })
630 )
631
632 return bytes;
633}
634EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe);
635#endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
636
576size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 637size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
577{ 638{
578 char *to = addr; 639 char *to = addr;
@@ -1012,7 +1073,7 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1012} 1073}
1013EXPORT_SYMBOL(iov_iter_gap_alignment); 1074EXPORT_SYMBOL(iov_iter_gap_alignment);
1014 1075
1015static inline size_t __pipe_get_pages(struct iov_iter *i, 1076static inline ssize_t __pipe_get_pages(struct iov_iter *i,
1016 size_t maxsize, 1077 size_t maxsize,
1017 struct page **pages, 1078 struct page **pages,
1018 int idx, 1079 int idx,
@@ -1102,7 +1163,7 @@ static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
1102 size_t *start) 1163 size_t *start)
1103{ 1164{
1104 struct page **p; 1165 struct page **p;
1105 size_t n; 1166 ssize_t n;
1106 int idx; 1167 int idx;
1107 int npages; 1168 int npages;
1108 1169
diff --git a/lib/kfifo.c b/lib/kfifo.c
index b0f757bf7213..015656aa8182 100644
--- a/lib/kfifo.c
+++ b/lib/kfifo.c
@@ -54,7 +54,7 @@ int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
54 return -EINVAL; 54 return -EINVAL;
55 } 55 }
56 56
57 fifo->data = kmalloc(size * esize, gfp_mask); 57 fifo->data = kmalloc_array(esize, size, gfp_mask);
58 58
59 if (!fifo->data) { 59 if (!fifo->data) {
60 fifo->mask = 0; 60 fifo->mask = 0;
diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c
index 15ea216a67ce..63d0816ab23b 100644
--- a/lib/kobject_uevent.c
+++ b/lib/kobject_uevent.c
@@ -22,6 +22,7 @@
22#include <linux/socket.h> 22#include <linux/socket.h>
23#include <linux/skbuff.h> 23#include <linux/skbuff.h>
24#include <linux/netlink.h> 24#include <linux/netlink.h>
25#include <linux/uidgid.h>
25#include <linux/uuid.h> 26#include <linux/uuid.h>
26#include <linux/ctype.h> 27#include <linux/ctype.h>
27#include <net/sock.h> 28#include <net/sock.h>
@@ -231,30 +232,6 @@ out:
231 return r; 232 return r;
232} 233}
233 234
234#ifdef CONFIG_NET
235static int kobj_bcast_filter(struct sock *dsk, struct sk_buff *skb, void *data)
236{
237 struct kobject *kobj = data, *ksobj;
238 const struct kobj_ns_type_operations *ops;
239
240 ops = kobj_ns_ops(kobj);
241 if (!ops && kobj->kset) {
242 ksobj = &kobj->kset->kobj;
243 if (ksobj->parent != NULL)
244 ops = kobj_ns_ops(ksobj->parent);
245 }
246
247 if (ops && ops->netlink_ns && kobj->ktype->namespace) {
248 const void *sock_ns, *ns;
249 ns = kobj->ktype->namespace(kobj);
250 sock_ns = ops->netlink_ns(dsk);
251 return sock_ns != ns;
252 }
253
254 return 0;
255}
256#endif
257
258#ifdef CONFIG_UEVENT_HELPER 235#ifdef CONFIG_UEVENT_HELPER
259static int kobj_usermode_filter(struct kobject *kobj) 236static int kobj_usermode_filter(struct kobject *kobj)
260{ 237{
@@ -296,15 +273,44 @@ static void cleanup_uevent_env(struct subprocess_info *info)
296} 273}
297#endif 274#endif
298 275
299static int kobject_uevent_net_broadcast(struct kobject *kobj, 276#ifdef CONFIG_NET
300 struct kobj_uevent_env *env, 277static struct sk_buff *alloc_uevent_skb(struct kobj_uevent_env *env,
301 const char *action_string, 278 const char *action_string,
302 const char *devpath) 279 const char *devpath)
303{ 280{
304 int retval = 0; 281 struct netlink_skb_parms *parms;
305#if defined(CONFIG_NET) 282 struct sk_buff *skb = NULL;
283 char *scratch;
284 size_t len;
285
286 /* allocate message with maximum possible size */
287 len = strlen(action_string) + strlen(devpath) + 2;
288 skb = alloc_skb(len + env->buflen, GFP_KERNEL);
289 if (!skb)
290 return NULL;
291
292 /* add header */
293 scratch = skb_put(skb, len);
294 sprintf(scratch, "%s@%s", action_string, devpath);
295
296 skb_put_data(skb, env->buf, env->buflen);
297
298 parms = &NETLINK_CB(skb);
299 parms->creds.uid = GLOBAL_ROOT_UID;
300 parms->creds.gid = GLOBAL_ROOT_GID;
301 parms->dst_group = 1;
302 parms->portid = 0;
303
304 return skb;
305}
306
307static int uevent_net_broadcast_untagged(struct kobj_uevent_env *env,
308 const char *action_string,
309 const char *devpath)
310{
306 struct sk_buff *skb = NULL; 311 struct sk_buff *skb = NULL;
307 struct uevent_sock *ue_sk; 312 struct uevent_sock *ue_sk;
313 int retval = 0;
308 314
309 /* send netlink message */ 315 /* send netlink message */
310 list_for_each_entry(ue_sk, &uevent_sock_list, list) { 316 list_for_each_entry(ue_sk, &uevent_sock_list, list) {
@@ -314,37 +320,99 @@ static int kobject_uevent_net_broadcast(struct kobject *kobj,
314 continue; 320 continue;
315 321
316 if (!skb) { 322 if (!skb) {
317 /* allocate message with the maximum possible size */
318 size_t len = strlen(action_string) + strlen(devpath) + 2;
319 char *scratch;
320
321 retval = -ENOMEM; 323 retval = -ENOMEM;
322 skb = alloc_skb(len + env->buflen, GFP_KERNEL); 324 skb = alloc_uevent_skb(env, action_string, devpath);
323 if (!skb) 325 if (!skb)
324 continue; 326 continue;
325
326 /* add header */
327 scratch = skb_put(skb, len);
328 sprintf(scratch, "%s@%s", action_string, devpath);
329
330 skb_put_data(skb, env->buf, env->buflen);
331
332 NETLINK_CB(skb).dst_group = 1;
333 } 327 }
334 328
335 retval = netlink_broadcast_filtered(uevent_sock, skb_get(skb), 329 retval = netlink_broadcast(uevent_sock, skb_get(skb), 0, 1,
336 0, 1, GFP_KERNEL, 330 GFP_KERNEL);
337 kobj_bcast_filter,
338 kobj);
339 /* ENOBUFS should be handled in userspace */ 331 /* ENOBUFS should be handled in userspace */
340 if (retval == -ENOBUFS || retval == -ESRCH) 332 if (retval == -ENOBUFS || retval == -ESRCH)
341 retval = 0; 333 retval = 0;
342 } 334 }
343 consume_skb(skb); 335 consume_skb(skb);
344#endif 336
345 return retval; 337 return retval;
346} 338}
347 339
340static int uevent_net_broadcast_tagged(struct sock *usk,
341 struct kobj_uevent_env *env,
342 const char *action_string,
343 const char *devpath)
344{
345 struct user_namespace *owning_user_ns = sock_net(usk)->user_ns;
346 struct sk_buff *skb = NULL;
347 int ret = 0;
348
349 skb = alloc_uevent_skb(env, action_string, devpath);
350 if (!skb)
351 return -ENOMEM;
352
353 /* fix credentials */
354 if (owning_user_ns != &init_user_ns) {
355 struct netlink_skb_parms *parms = &NETLINK_CB(skb);
356 kuid_t root_uid;
357 kgid_t root_gid;
358
359 /* fix uid */
360 root_uid = make_kuid(owning_user_ns, 0);
361 if (uid_valid(root_uid))
362 parms->creds.uid = root_uid;
363
364 /* fix gid */
365 root_gid = make_kgid(owning_user_ns, 0);
366 if (gid_valid(root_gid))
367 parms->creds.gid = root_gid;
368 }
369
370 ret = netlink_broadcast(usk, skb, 0, 1, GFP_KERNEL);
371 /* ENOBUFS should be handled in userspace */
372 if (ret == -ENOBUFS || ret == -ESRCH)
373 ret = 0;
374
375 return ret;
376}
377#endif
378
379static int kobject_uevent_net_broadcast(struct kobject *kobj,
380 struct kobj_uevent_env *env,
381 const char *action_string,
382 const char *devpath)
383{
384 int ret = 0;
385
386#ifdef CONFIG_NET
387 const struct kobj_ns_type_operations *ops;
388 const struct net *net = NULL;
389
390 ops = kobj_ns_ops(kobj);
391 if (!ops && kobj->kset) {
392 struct kobject *ksobj = &kobj->kset->kobj;
393 if (ksobj->parent != NULL)
394 ops = kobj_ns_ops(ksobj->parent);
395 }
396
397 /* kobjects currently only carry network namespace tags and they
398 * are the only tag relevant here since we want to decide which
399 * network namespaces to broadcast the uevent into.
400 */
401 if (ops && ops->netlink_ns && kobj->ktype->namespace)
402 if (ops->type == KOBJ_NS_TYPE_NET)
403 net = kobj->ktype->namespace(kobj);
404
405 if (!net)
406 ret = uevent_net_broadcast_untagged(env, action_string,
407 devpath);
408 else
409 ret = uevent_net_broadcast_tagged(net->uevent_sock->sk, env,
410 action_string, devpath);
411#endif
412
413 return ret;
414}
415
348static void zap_modalias_env(struct kobj_uevent_env *env) 416static void zap_modalias_env(struct kobj_uevent_env *env)
349{ 417{
350 static const char modalias_prefix[] = "MODALIAS="; 418 static const char modalias_prefix[] = "MODALIAS=";
@@ -703,9 +771,13 @@ static int uevent_net_init(struct net *net)
703 771
704 net->uevent_sock = ue_sk; 772 net->uevent_sock = ue_sk;
705 773
706 mutex_lock(&uevent_sock_mutex); 774 /* Restrict uevents to initial user namespace. */
707 list_add_tail(&ue_sk->list, &uevent_sock_list); 775 if (sock_net(ue_sk->sk)->user_ns == &init_user_ns) {
708 mutex_unlock(&uevent_sock_mutex); 776 mutex_lock(&uevent_sock_mutex);
777 list_add_tail(&ue_sk->list, &uevent_sock_list);
778 mutex_unlock(&uevent_sock_mutex);
779 }
780
709 return 0; 781 return 0;
710} 782}
711 783
@@ -713,9 +785,11 @@ static void uevent_net_exit(struct net *net)
713{ 785{
714 struct uevent_sock *ue_sk = net->uevent_sock; 786 struct uevent_sock *ue_sk = net->uevent_sock;
715 787
716 mutex_lock(&uevent_sock_mutex); 788 if (sock_net(ue_sk->sk)->user_ns == &init_user_ns) {
717 list_del(&ue_sk->list); 789 mutex_lock(&uevent_sock_mutex);
718 mutex_unlock(&uevent_sock_mutex); 790 list_del(&ue_sk->list);
791 mutex_unlock(&uevent_sock_mutex);
792 }
719 793
720 netlink_kernel_release(ue_sk->sk); 794 netlink_kernel_release(ue_sk->sk);
721 kfree(ue_sk); 795 kfree(ue_sk);
diff --git a/lib/lru_cache.c b/lib/lru_cache.c
index 28ba40b99337..2b10a4024c35 100644
--- a/lib/lru_cache.c
+++ b/lib/lru_cache.c
@@ -119,7 +119,7 @@ struct lru_cache *lc_create(const char *name, struct kmem_cache *cache,
119 slot = kcalloc(e_count, sizeof(struct hlist_head), GFP_KERNEL); 119 slot = kcalloc(e_count, sizeof(struct hlist_head), GFP_KERNEL);
120 if (!slot) 120 if (!slot)
121 goto out_fail; 121 goto out_fail;
122 element = kzalloc(e_count * sizeof(struct lc_element *), GFP_KERNEL); 122 element = kcalloc(e_count, sizeof(struct lc_element *), GFP_KERNEL);
123 if (!element) 123 if (!element)
124 goto out_fail; 124 goto out_fail;
125 125
diff --git a/lib/mpi/mpi-internal.h b/lib/mpi/mpi-internal.h
index 7eceeddb3fb8..c2d6f4efcfbc 100644
--- a/lib/mpi/mpi-internal.h
+++ b/lib/mpi/mpi-internal.h
@@ -65,13 +65,6 @@
65typedef mpi_limb_t *mpi_ptr_t; /* pointer to a limb */ 65typedef mpi_limb_t *mpi_ptr_t; /* pointer to a limb */
66typedef int mpi_size_t; /* (must be a signed type) */ 66typedef int mpi_size_t; /* (must be a signed type) */
67 67
68static inline int RESIZE_IF_NEEDED(MPI a, unsigned b)
69{
70 if (a->alloced < b)
71 return mpi_resize(a, b);
72 return 0;
73}
74
75/* Copy N limbs from S to D. */ 68/* Copy N limbs from S to D. */
76#define MPN_COPY(d, s, n) \ 69#define MPN_COPY(d, s, n) \
77 do { \ 70 do { \
@@ -80,13 +73,6 @@ static inline int RESIZE_IF_NEEDED(MPI a, unsigned b)
80 (d)[_i] = (s)[_i]; \ 73 (d)[_i] = (s)[_i]; \
81 } while (0) 74 } while (0)
82 75
83#define MPN_COPY_INCR(d, s, n) \
84 do { \
85 mpi_size_t _i; \
86 for (_i = 0; _i < (n); _i++) \
87 (d)[_i] = (s)[_i]; \
88 } while (0)
89
90#define MPN_COPY_DECR(d, s, n) \ 76#define MPN_COPY_DECR(d, s, n) \
91 do { \ 77 do { \
92 mpi_size_t _i; \ 78 mpi_size_t _i; \
@@ -111,15 +97,6 @@ static inline int RESIZE_IF_NEEDED(MPI a, unsigned b)
111 } \ 97 } \
112 } while (0) 98 } while (0)
113 99
114#define MPN_NORMALIZE_NOT_ZERO(d, n) \
115 do { \
116 for (;;) { \
117 if ((d)[(n)-1]) \
118 break; \
119 (n)--; \
120 } \
121 } while (0)
122
123#define MPN_MUL_N_RECURSE(prodp, up, vp, size, tspace) \ 100#define MPN_MUL_N_RECURSE(prodp, up, vp, size, tspace) \
124 do { \ 101 do { \
125 if ((size) < KARATSUBA_THRESHOLD) \ 102 if ((size) < KARATSUBA_THRESHOLD) \
@@ -128,46 +105,11 @@ static inline int RESIZE_IF_NEEDED(MPI a, unsigned b)
128 mul_n(prodp, up, vp, size, tspace); \ 105 mul_n(prodp, up, vp, size, tspace); \
129 } while (0); 106 } while (0);
130 107
131/* Divide the two-limb number in (NH,,NL) by D, with DI being the largest
132 * limb not larger than (2**(2*BITS_PER_MP_LIMB))/D - (2**BITS_PER_MP_LIMB).
133 * If this would yield overflow, DI should be the largest possible number
134 * (i.e., only ones). For correct operation, the most significant bit of D
135 * has to be set. Put the quotient in Q and the remainder in R.
136 */
137#define UDIV_QRNND_PREINV(q, r, nh, nl, d, di) \
138 do { \
139 mpi_limb_t _q, _ql, _r; \
140 mpi_limb_t _xh, _xl; \
141 umul_ppmm(_q, _ql, (nh), (di)); \
142 _q += (nh); /* DI is 2**BITS_PER_MPI_LIMB too small */ \
143 umul_ppmm(_xh, _xl, _q, (d)); \
144 sub_ddmmss(_xh, _r, (nh), (nl), _xh, _xl); \
145 if (_xh) { \
146 sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \
147 _q++; \
148 if (_xh) { \
149 sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \
150 _q++; \
151 } \
152 } \
153 if (_r >= (d)) { \
154 _r -= (d); \
155 _q++; \
156 } \
157 (r) = _r; \
158 (q) = _q; \
159 } while (0)
160
161/*-- mpiutil.c --*/ 108/*-- mpiutil.c --*/
162mpi_ptr_t mpi_alloc_limb_space(unsigned nlimbs); 109mpi_ptr_t mpi_alloc_limb_space(unsigned nlimbs);
163void mpi_free_limb_space(mpi_ptr_t a); 110void mpi_free_limb_space(mpi_ptr_t a);
164void mpi_assign_limb_space(MPI a, mpi_ptr_t ap, unsigned nlimbs); 111void mpi_assign_limb_space(MPI a, mpi_ptr_t ap, unsigned nlimbs);
165 112
166/*-- mpi-bit.c --*/
167void mpi_rshift_limbs(MPI a, unsigned int count);
168int mpi_lshift_limbs(MPI a, unsigned int count);
169
170/*-- mpihelp-add.c --*/
171static inline mpi_limb_t mpihelp_add_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, 113static inline mpi_limb_t mpihelp_add_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
172 mpi_size_t s1_size, mpi_limb_t s2_limb); 114 mpi_size_t s1_size, mpi_limb_t s2_limb);
173mpi_limb_t mpihelp_add_n(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, 115mpi_limb_t mpihelp_add_n(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
@@ -175,7 +117,6 @@ mpi_limb_t mpihelp_add_n(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
175static inline mpi_limb_t mpihelp_add(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size, 117static inline mpi_limb_t mpihelp_add(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size,
176 mpi_ptr_t s2_ptr, mpi_size_t s2_size); 118 mpi_ptr_t s2_ptr, mpi_size_t s2_size);
177 119
178/*-- mpihelp-sub.c --*/
179static inline mpi_limb_t mpihelp_sub_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, 120static inline mpi_limb_t mpihelp_sub_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
180 mpi_size_t s1_size, mpi_limb_t s2_limb); 121 mpi_size_t s1_size, mpi_limb_t s2_limb);
181mpi_limb_t mpihelp_sub_n(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, 122mpi_limb_t mpihelp_sub_n(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
@@ -183,10 +124,10 @@ mpi_limb_t mpihelp_sub_n(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
183static inline mpi_limb_t mpihelp_sub(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size, 124static inline mpi_limb_t mpihelp_sub(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, mpi_size_t s1_size,
184 mpi_ptr_t s2_ptr, mpi_size_t s2_size); 125 mpi_ptr_t s2_ptr, mpi_size_t s2_size);
185 126
186/*-- mpihelp-cmp.c --*/ 127/*-- mpih-cmp.c --*/
187int mpihelp_cmp(mpi_ptr_t op1_ptr, mpi_ptr_t op2_ptr, mpi_size_t size); 128int mpihelp_cmp(mpi_ptr_t op1_ptr, mpi_ptr_t op2_ptr, mpi_size_t size);
188 129
189/*-- mpihelp-mul.c --*/ 130/*-- mpih-mul.c --*/
190 131
191struct karatsuba_ctx { 132struct karatsuba_ctx {
192 struct karatsuba_ctx *next; 133 struct karatsuba_ctx *next;
@@ -202,7 +143,6 @@ mpi_limb_t mpihelp_addmul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
202 mpi_size_t s1_size, mpi_limb_t s2_limb); 143 mpi_size_t s1_size, mpi_limb_t s2_limb);
203mpi_limb_t mpihelp_submul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, 144mpi_limb_t mpihelp_submul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
204 mpi_size_t s1_size, mpi_limb_t s2_limb); 145 mpi_size_t s1_size, mpi_limb_t s2_limb);
205int mpihelp_mul_n(mpi_ptr_t prodp, mpi_ptr_t up, mpi_ptr_t vp, mpi_size_t size);
206int mpihelp_mul(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize, 146int mpihelp_mul(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t usize,
207 mpi_ptr_t vp, mpi_size_t vsize, mpi_limb_t *_result); 147 mpi_ptr_t vp, mpi_size_t vsize, mpi_limb_t *_result);
208void mpih_sqr_n_basecase(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size); 148void mpih_sqr_n_basecase(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size);
@@ -214,21 +154,16 @@ int mpihelp_mul_karatsuba_case(mpi_ptr_t prodp,
214 mpi_ptr_t vp, mpi_size_t vsize, 154 mpi_ptr_t vp, mpi_size_t vsize,
215 struct karatsuba_ctx *ctx); 155 struct karatsuba_ctx *ctx);
216 156
217/*-- mpihelp-mul_1.c (or xxx/cpu/ *.S) --*/ 157/*-- generic_mpih-mul1.c --*/
218mpi_limb_t mpihelp_mul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr, 158mpi_limb_t mpihelp_mul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
219 mpi_size_t s1_size, mpi_limb_t s2_limb); 159 mpi_size_t s1_size, mpi_limb_t s2_limb);
220 160
221/*-- mpihelp-div.c --*/ 161/*-- mpih-div.c --*/
222mpi_limb_t mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
223 mpi_limb_t divisor_limb);
224mpi_limb_t mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs, 162mpi_limb_t mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs,
225 mpi_ptr_t np, mpi_size_t nsize, 163 mpi_ptr_t np, mpi_size_t nsize,
226 mpi_ptr_t dp, mpi_size_t dsize); 164 mpi_ptr_t dp, mpi_size_t dsize);
227mpi_limb_t mpihelp_divmod_1(mpi_ptr_t quot_ptr,
228 mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
229 mpi_limb_t divisor_limb);
230 165
231/*-- mpihelp-shift.c --*/ 166/*-- generic_mpih-[lr]shift.c --*/
232mpi_limb_t mpihelp_lshift(mpi_ptr_t wp, mpi_ptr_t up, mpi_size_t usize, 167mpi_limb_t mpihelp_lshift(mpi_ptr_t wp, mpi_ptr_t up, mpi_size_t usize,
233 unsigned cnt); 168 unsigned cnt);
234mpi_limb_t mpihelp_rshift(mpi_ptr_t wp, mpi_ptr_t up, mpi_size_t usize, 169mpi_limb_t mpihelp_rshift(mpi_ptr_t wp, mpi_ptr_t up, mpi_size_t usize,
diff --git a/lib/mpi/mpiutil.c b/lib/mpi/mpiutil.c
index 314f4dfa603e..20ed0f766787 100644
--- a/lib/mpi/mpiutil.c
+++ b/lib/mpi/mpiutil.c
@@ -91,14 +91,14 @@ int mpi_resize(MPI a, unsigned nlimbs)
91 return 0; /* no need to do it */ 91 return 0; /* no need to do it */
92 92
93 if (a->d) { 93 if (a->d) {
94 p = kmalloc(nlimbs * sizeof(mpi_limb_t), GFP_KERNEL); 94 p = kmalloc_array(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL);
95 if (!p) 95 if (!p)
96 return -ENOMEM; 96 return -ENOMEM;
97 memcpy(p, a->d, a->alloced * sizeof(mpi_limb_t)); 97 memcpy(p, a->d, a->alloced * sizeof(mpi_limb_t));
98 kzfree(a->d); 98 kzfree(a->d);
99 a->d = p; 99 a->d = p;
100 } else { 100 } else {
101 a->d = kzalloc(nlimbs * sizeof(mpi_limb_t), GFP_KERNEL); 101 a->d = kcalloc(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL);
102 if (!a->d) 102 if (!a->d)
103 return -ENOMEM; 103 return -ENOMEM;
104 } 104 }
diff --git a/lib/percpu_ida.c b/lib/percpu_ida.c
index 6016f1deb1f5..beb14839b41a 100644
--- a/lib/percpu_ida.c
+++ b/lib/percpu_ida.c
@@ -112,18 +112,6 @@ static inline void alloc_global_tags(struct percpu_ida *pool,
112 min(pool->nr_free, pool->percpu_batch_size)); 112 min(pool->nr_free, pool->percpu_batch_size));
113} 113}
114 114
115static inline unsigned alloc_local_tag(struct percpu_ida_cpu *tags)
116{
117 int tag = -ENOSPC;
118
119 spin_lock(&tags->lock);
120 if (tags->nr_free)
121 tag = tags->freelist[--tags->nr_free];
122 spin_unlock(&tags->lock);
123
124 return tag;
125}
126
127/** 115/**
128 * percpu_ida_alloc - allocate a tag 116 * percpu_ida_alloc - allocate a tag
129 * @pool: pool to allocate from 117 * @pool: pool to allocate from
@@ -147,20 +135,22 @@ int percpu_ida_alloc(struct percpu_ida *pool, int state)
147 DEFINE_WAIT(wait); 135 DEFINE_WAIT(wait);
148 struct percpu_ida_cpu *tags; 136 struct percpu_ida_cpu *tags;
149 unsigned long flags; 137 unsigned long flags;
150 int tag; 138 int tag = -ENOSPC;
151 139
152 local_irq_save(flags); 140 tags = raw_cpu_ptr(pool->tag_cpu);
153 tags = this_cpu_ptr(pool->tag_cpu); 141 spin_lock_irqsave(&tags->lock, flags);
154 142
155 /* Fastpath */ 143 /* Fastpath */
156 tag = alloc_local_tag(tags); 144 if (likely(tags->nr_free)) {
157 if (likely(tag >= 0)) { 145 tag = tags->freelist[--tags->nr_free];
158 local_irq_restore(flags); 146 spin_unlock_irqrestore(&tags->lock, flags);
159 return tag; 147 return tag;
160 } 148 }
149 spin_unlock_irqrestore(&tags->lock, flags);
161 150
162 while (1) { 151 while (1) {
163 spin_lock(&pool->lock); 152 spin_lock_irqsave(&pool->lock, flags);
153 tags = this_cpu_ptr(pool->tag_cpu);
164 154
165 /* 155 /*
166 * prepare_to_wait() must come before steal_tags(), in case 156 * prepare_to_wait() must come before steal_tags(), in case
@@ -184,8 +174,7 @@ int percpu_ida_alloc(struct percpu_ida *pool, int state)
184 &pool->cpus_have_tags); 174 &pool->cpus_have_tags);
185 } 175 }
186 176
187 spin_unlock(&pool->lock); 177 spin_unlock_irqrestore(&pool->lock, flags);
188 local_irq_restore(flags);
189 178
190 if (tag >= 0 || state == TASK_RUNNING) 179 if (tag >= 0 || state == TASK_RUNNING)
191 break; 180 break;
@@ -196,9 +185,6 @@ int percpu_ida_alloc(struct percpu_ida *pool, int state)
196 } 185 }
197 186
198 schedule(); 187 schedule();
199
200 local_irq_save(flags);
201 tags = this_cpu_ptr(pool->tag_cpu);
202 } 188 }
203 if (state != TASK_RUNNING) 189 if (state != TASK_RUNNING)
204 finish_wait(&pool->wait, &wait); 190 finish_wait(&pool->wait, &wait);
@@ -222,28 +208,24 @@ void percpu_ida_free(struct percpu_ida *pool, unsigned tag)
222 208
223 BUG_ON(tag >= pool->nr_tags); 209 BUG_ON(tag >= pool->nr_tags);
224 210
225 local_irq_save(flags); 211 tags = raw_cpu_ptr(pool->tag_cpu);
226 tags = this_cpu_ptr(pool->tag_cpu);
227 212
228 spin_lock(&tags->lock); 213 spin_lock_irqsave(&tags->lock, flags);
229 tags->freelist[tags->nr_free++] = tag; 214 tags->freelist[tags->nr_free++] = tag;
230 215
231 nr_free = tags->nr_free; 216 nr_free = tags->nr_free;
232 spin_unlock(&tags->lock);
233 217
234 if (nr_free == 1) { 218 if (nr_free == 1) {
235 cpumask_set_cpu(smp_processor_id(), 219 cpumask_set_cpu(smp_processor_id(),
236 &pool->cpus_have_tags); 220 &pool->cpus_have_tags);
237 wake_up(&pool->wait); 221 wake_up(&pool->wait);
238 } 222 }
223 spin_unlock_irqrestore(&tags->lock, flags);
239 224
240 if (nr_free == pool->percpu_max_size) { 225 if (nr_free == pool->percpu_max_size) {
241 spin_lock(&pool->lock); 226 spin_lock_irqsave(&pool->lock, flags);
227 spin_lock(&tags->lock);
242 228
243 /*
244 * Global lock held and irqs disabled, don't need percpu
245 * lock
246 */
247 if (tags->nr_free == pool->percpu_max_size) { 229 if (tags->nr_free == pool->percpu_max_size) {
248 move_tags(pool->freelist, &pool->nr_free, 230 move_tags(pool->freelist, &pool->nr_free,
249 tags->freelist, &tags->nr_free, 231 tags->freelist, &tags->nr_free,
@@ -251,10 +233,9 @@ void percpu_ida_free(struct percpu_ida *pool, unsigned tag)
251 233
252 wake_up(&pool->wait); 234 wake_up(&pool->wait);
253 } 235 }
254 spin_unlock(&pool->lock); 236 spin_unlock(&tags->lock);
237 spin_unlock_irqrestore(&pool->lock, flags);
255 } 238 }
256
257 local_irq_restore(flags);
258} 239}
259EXPORT_SYMBOL_GPL(percpu_ida_free); 240EXPORT_SYMBOL_GPL(percpu_ida_free);
260 241
@@ -346,29 +327,27 @@ int percpu_ida_for_each_free(struct percpu_ida *pool, percpu_ida_cb fn,
346 struct percpu_ida_cpu *remote; 327 struct percpu_ida_cpu *remote;
347 unsigned cpu, i, err = 0; 328 unsigned cpu, i, err = 0;
348 329
349 local_irq_save(flags);
350 for_each_possible_cpu(cpu) { 330 for_each_possible_cpu(cpu) {
351 remote = per_cpu_ptr(pool->tag_cpu, cpu); 331 remote = per_cpu_ptr(pool->tag_cpu, cpu);
352 spin_lock(&remote->lock); 332 spin_lock_irqsave(&remote->lock, flags);
353 for (i = 0; i < remote->nr_free; i++) { 333 for (i = 0; i < remote->nr_free; i++) {
354 err = fn(remote->freelist[i], data); 334 err = fn(remote->freelist[i], data);
355 if (err) 335 if (err)
356 break; 336 break;
357 } 337 }
358 spin_unlock(&remote->lock); 338 spin_unlock_irqrestore(&remote->lock, flags);
359 if (err) 339 if (err)
360 goto out; 340 goto out;
361 } 341 }
362 342
363 spin_lock(&pool->lock); 343 spin_lock_irqsave(&pool->lock, flags);
364 for (i = 0; i < pool->nr_free; i++) { 344 for (i = 0; i < pool->nr_free; i++) {
365 err = fn(pool->freelist[i], data); 345 err = fn(pool->freelist[i], data);
366 if (err) 346 if (err)
367 break; 347 break;
368 } 348 }
369 spin_unlock(&pool->lock); 349 spin_unlock_irqrestore(&pool->lock, flags);
370out: 350out:
371 local_irq_restore(flags);
372 return err; 351 return err;
373} 352}
374EXPORT_SYMBOL_GPL(percpu_ida_for_each_free); 353EXPORT_SYMBOL_GPL(percpu_ida_for_each_free);
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index da9e10c827df..a9e41aed6de4 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -1612,11 +1612,9 @@ static void set_iter_tags(struct radix_tree_iter *iter,
1612static void __rcu **skip_siblings(struct radix_tree_node **nodep, 1612static void __rcu **skip_siblings(struct radix_tree_node **nodep,
1613 void __rcu **slot, struct radix_tree_iter *iter) 1613 void __rcu **slot, struct radix_tree_iter *iter)
1614{ 1614{
1615 void *sib = node_to_entry(slot - 1);
1616
1617 while (iter->index < iter->next_index) { 1615 while (iter->index < iter->next_index) {
1618 *nodep = rcu_dereference_raw(*slot); 1616 *nodep = rcu_dereference_raw(*slot);
1619 if (*nodep && *nodep != sib) 1617 if (*nodep && !is_sibling_entry(iter->node, *nodep))
1620 return slot; 1618 return slot;
1621 slot++; 1619 slot++;
1622 iter->index = __radix_tree_iter_add(iter, 1); 1620 iter->index = __radix_tree_iter_add(iter, 1);
@@ -1631,7 +1629,7 @@ void __rcu **__radix_tree_next_slot(void __rcu **slot,
1631 struct radix_tree_iter *iter, unsigned flags) 1629 struct radix_tree_iter *iter, unsigned flags)
1632{ 1630{
1633 unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK; 1631 unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
1634 struct radix_tree_node *node = rcu_dereference_raw(*slot); 1632 struct radix_tree_node *node;
1635 1633
1636 slot = skip_siblings(&node, slot, iter); 1634 slot = skip_siblings(&node, slot, iter);
1637 1635
@@ -2036,10 +2034,12 @@ void *radix_tree_delete_item(struct radix_tree_root *root,
2036 unsigned long index, void *item) 2034 unsigned long index, void *item)
2037{ 2035{
2038 struct radix_tree_node *node = NULL; 2036 struct radix_tree_node *node = NULL;
2039 void __rcu **slot; 2037 void __rcu **slot = NULL;
2040 void *entry; 2038 void *entry;
2041 2039
2042 entry = __radix_tree_lookup(root, index, &node, &slot); 2040 entry = __radix_tree_lookup(root, index, &node, &slot);
2041 if (!slot)
2042 return NULL;
2043 if (!entry && (!is_idr(root) || node_tag_get(root, node, IDR_FREE, 2043 if (!entry && (!is_idr(root) || node_tag_get(root, node, IDR_FREE,
2044 get_slot_offset(node, slot)))) 2044 get_slot_offset(node, slot))))
2045 return NULL; 2045 return NULL;
diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c
index 7d36c1e27ff6..b7055b2a07d3 100644
--- a/lib/rbtree_test.c
+++ b/lib/rbtree_test.c
@@ -247,7 +247,7 @@ static int __init rbtree_test_init(void)
247 cycles_t time1, time2, time; 247 cycles_t time1, time2, time;
248 struct rb_node *node; 248 struct rb_node *node;
249 249
250 nodes = kmalloc(nnodes * sizeof(*nodes), GFP_KERNEL); 250 nodes = kmalloc_array(nnodes, sizeof(*nodes), GFP_KERNEL);
251 if (!nodes) 251 if (!nodes)
252 return -ENOMEM; 252 return -ENOMEM;
253 253
diff --git a/lib/reed_solomon/decode_rs.c b/lib/reed_solomon/decode_rs.c
index 0ec3f257ffdf..1db74eb098d0 100644
--- a/lib/reed_solomon/decode_rs.c
+++ b/lib/reed_solomon/decode_rs.c
@@ -1,22 +1,16 @@
1// SPDX-License-Identifier: GPL-2.0
1/* 2/*
2 * lib/reed_solomon/decode_rs.c 3 * Generic Reed Solomon encoder / decoder library
3 *
4 * Overview:
5 * Generic Reed Solomon encoder / decoder library
6 * 4 *
7 * Copyright 2002, Phil Karn, KA9Q 5 * Copyright 2002, Phil Karn, KA9Q
8 * May be used under the terms of the GNU General Public License (GPL) 6 * May be used under the terms of the GNU General Public License (GPL)
9 * 7 *
10 * Adaption to the kernel by Thomas Gleixner (tglx@linutronix.de) 8 * Adaption to the kernel by Thomas Gleixner (tglx@linutronix.de)
11 * 9 *
12 * $Id: decode_rs.c,v 1.7 2005/11/07 11:14:59 gleixner Exp $ 10 * Generic data width independent code which is included by the wrappers.
13 *
14 */
15
16/* Generic data width independent code which is included by the
17 * wrappers.
18 */ 11 */
19{ 12{
13 struct rs_codec *rs = rsc->codec;
20 int deg_lambda, el, deg_omega; 14 int deg_lambda, el, deg_omega;
21 int i, j, r, k, pad; 15 int i, j, r, k, pad;
22 int nn = rs->nn; 16 int nn = rs->nn;
@@ -27,16 +21,22 @@
27 uint16_t *alpha_to = rs->alpha_to; 21 uint16_t *alpha_to = rs->alpha_to;
28 uint16_t *index_of = rs->index_of; 22 uint16_t *index_of = rs->index_of;
29 uint16_t u, q, tmp, num1, num2, den, discr_r, syn_error; 23 uint16_t u, q, tmp, num1, num2, den, discr_r, syn_error;
30 /* Err+Eras Locator poly and syndrome poly The maximum value
31 * of nroots is 8. So the necessary stack size will be about
32 * 220 bytes max.
33 */
34 uint16_t lambda[nroots + 1], syn[nroots];
35 uint16_t b[nroots + 1], t[nroots + 1], omega[nroots + 1];
36 uint16_t root[nroots], reg[nroots + 1], loc[nroots];
37 int count = 0; 24 int count = 0;
38 uint16_t msk = (uint16_t) rs->nn; 25 uint16_t msk = (uint16_t) rs->nn;
39 26
27 /*
28 * The decoder buffers are in the rs control struct. They are
29 * arrays sized [nroots + 1]
30 */
31 uint16_t *lambda = rsc->buffers + RS_DECODE_LAMBDA * (nroots + 1);
32 uint16_t *syn = rsc->buffers + RS_DECODE_SYN * (nroots + 1);
33 uint16_t *b = rsc->buffers + RS_DECODE_B * (nroots + 1);
34 uint16_t *t = rsc->buffers + RS_DECODE_T * (nroots + 1);
35 uint16_t *omega = rsc->buffers + RS_DECODE_OMEGA * (nroots + 1);
36 uint16_t *root = rsc->buffers + RS_DECODE_ROOT * (nroots + 1);
37 uint16_t *reg = rsc->buffers + RS_DECODE_REG * (nroots + 1);
38 uint16_t *loc = rsc->buffers + RS_DECODE_LOC * (nroots + 1);
39
40 /* Check length parameter for validity */ 40 /* Check length parameter for validity */
41 pad = nn - nroots - len; 41 pad = nn - nroots - len;
42 BUG_ON(pad < 0 || pad >= nn); 42 BUG_ON(pad < 0 || pad >= nn);
diff --git a/lib/reed_solomon/encode_rs.c b/lib/reed_solomon/encode_rs.c
index 0b5b1a6728ec..9112d46e869e 100644
--- a/lib/reed_solomon/encode_rs.c
+++ b/lib/reed_solomon/encode_rs.c
@@ -1,23 +1,16 @@
1// SPDX-License-Identifier: GPL-2.0
1/* 2/*
2 * lib/reed_solomon/encode_rs.c 3 * Generic Reed Solomon encoder / decoder library
3 *
4 * Overview:
5 * Generic Reed Solomon encoder / decoder library
6 * 4 *
7 * Copyright 2002, Phil Karn, KA9Q 5 * Copyright 2002, Phil Karn, KA9Q
8 * May be used under the terms of the GNU General Public License (GPL) 6 * May be used under the terms of the GNU General Public License (GPL)
9 * 7 *
10 * Adaption to the kernel by Thomas Gleixner (tglx@linutronix.de) 8 * Adaption to the kernel by Thomas Gleixner (tglx@linutronix.de)
11 * 9 *
12 * $Id: encode_rs.c,v 1.5 2005/11/07 11:14:59 gleixner Exp $ 10 * Generic data width independent code which is included by the wrappers.
13 *
14 */
15
16/* Generic data width independent code which is included by the
17 * wrappers.
18 * int encode_rsX (struct rs_control *rs, uintX_t *data, int len, uintY_t *par)
19 */ 11 */
20{ 12{
13 struct rs_codec *rs = rsc->codec;
21 int i, j, pad; 14 int i, j, pad;
22 int nn = rs->nn; 15 int nn = rs->nn;
23 int nroots = rs->nroots; 16 int nroots = rs->nroots;
diff --git a/lib/reed_solomon/reed_solomon.c b/lib/reed_solomon/reed_solomon.c
index 06d04cfa9339..d8bb1a1eba72 100644
--- a/lib/reed_solomon/reed_solomon.c
+++ b/lib/reed_solomon/reed_solomon.c
@@ -1,43 +1,34 @@
1// SPDX-License-Identifier: GPL-2.0
1/* 2/*
2 * lib/reed_solomon/reed_solomon.c 3 * Generic Reed Solomon encoder / decoder library
3 *
4 * Overview:
5 * Generic Reed Solomon encoder / decoder library
6 * 4 *
7 * Copyright (C) 2004 Thomas Gleixner (tglx@linutronix.de) 5 * Copyright (C) 2004 Thomas Gleixner (tglx@linutronix.de)
8 * 6 *
9 * Reed Solomon code lifted from reed solomon library written by Phil Karn 7 * Reed Solomon code lifted from reed solomon library written by Phil Karn
10 * Copyright 2002 Phil Karn, KA9Q 8 * Copyright 2002 Phil Karn, KA9Q
11 * 9 *
12 * $Id: rslib.c,v 1.7 2005/11/07 11:14:59 gleixner Exp $
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License version 2 as
16 * published by the Free Software Foundation.
17 *
18 * Description: 10 * Description:
19 * 11 *
20 * The generic Reed Solomon library provides runtime configurable 12 * The generic Reed Solomon library provides runtime configurable
21 * encoding / decoding of RS codes. 13 * encoding / decoding of RS codes.
22 * Each user must call init_rs to get a pointer to a rs_control
23 * structure for the given rs parameters. This structure is either
24 * generated or a already available matching control structure is used.
25 * If a structure is generated then the polynomial arrays for
26 * fast encoding / decoding are built. This can take some time so
27 * make sure not to call this function from a time critical path.
28 * Usually a module / driver should initialize the necessary
29 * rs_control structure on module / driver init and release it
30 * on exit.
31 * The encoding puts the calculated syndrome into a given syndrome
32 * buffer.
33 * The decoding is a two step process. The first step calculates
34 * the syndrome over the received (data + syndrome) and calls the
35 * second stage, which does the decoding / error correction itself.
36 * Many hw encoders provide a syndrome calculation over the received
37 * data + syndrome and can call the second stage directly.
38 * 14 *
15 * Each user must call init_rs to get a pointer to a rs_control structure
16 * for the given rs parameters. The control struct is unique per instance.
17 * It points to a codec which can be shared by multiple control structures.
18 * If a codec is newly allocated then the polynomial arrays for fast
19 * encoding / decoding are built. This can take some time so make sure not
20 * to call this function from a time critical path. Usually a module /
21 * driver should initialize the necessary rs_control structure on module /
22 * driver init and release it on exit.
23 *
24 * The encoding puts the calculated syndrome into a given syndrome buffer.
25 *
26 * The decoding is a two step process. The first step calculates the
27 * syndrome over the received (data + syndrome) and calls the second stage,
28 * which does the decoding / error correction itself. Many hw encoders
29 * provide a syndrome calculation over the received data + syndrome and can
30 * call the second stage directly.
39 */ 31 */
40
41#include <linux/errno.h> 32#include <linux/errno.h>
42#include <linux/kernel.h> 33#include <linux/kernel.h>
43#include <linux/init.h> 34#include <linux/init.h>
@@ -46,32 +37,44 @@
46#include <linux/slab.h> 37#include <linux/slab.h>
47#include <linux/mutex.h> 38#include <linux/mutex.h>
48 39
49/* This list holds all currently allocated rs control structures */ 40enum {
50static LIST_HEAD (rslist); 41 RS_DECODE_LAMBDA,
42 RS_DECODE_SYN,
43 RS_DECODE_B,
44 RS_DECODE_T,
45 RS_DECODE_OMEGA,
46 RS_DECODE_ROOT,
47 RS_DECODE_REG,
48 RS_DECODE_LOC,
49 RS_DECODE_NUM_BUFFERS
50};
51
52/* This list holds all currently allocated rs codec structures */
53static LIST_HEAD(codec_list);
51/* Protection for the list */ 54/* Protection for the list */
52static DEFINE_MUTEX(rslistlock); 55static DEFINE_MUTEX(rslistlock);
53 56
54/** 57/**
55 * rs_init - Initialize a Reed-Solomon codec 58 * codec_init - Initialize a Reed-Solomon codec
56 * @symsize: symbol size, bits (1-8) 59 * @symsize: symbol size, bits (1-8)
57 * @gfpoly: Field generator polynomial coefficients 60 * @gfpoly: Field generator polynomial coefficients
58 * @gffunc: Field generator function 61 * @gffunc: Field generator function
59 * @fcr: first root of RS code generator polynomial, index form 62 * @fcr: first root of RS code generator polynomial, index form
60 * @prim: primitive element to generate polynomial roots 63 * @prim: primitive element to generate polynomial roots
61 * @nroots: RS code generator polynomial degree (number of roots) 64 * @nroots: RS code generator polynomial degree (number of roots)
65 * @gfp: GFP_ flags for allocations
62 * 66 *
63 * Allocate a control structure and the polynom arrays for faster 67 * Allocate a codec structure and the polynom arrays for faster
64 * en/decoding. Fill the arrays according to the given parameters. 68 * en/decoding. Fill the arrays according to the given parameters.
65 */ 69 */
66static struct rs_control *rs_init(int symsize, int gfpoly, int (*gffunc)(int), 70static struct rs_codec *codec_init(int symsize, int gfpoly, int (*gffunc)(int),
67 int fcr, int prim, int nroots) 71 int fcr, int prim, int nroots, gfp_t gfp)
68{ 72{
69 struct rs_control *rs;
70 int i, j, sr, root, iprim; 73 int i, j, sr, root, iprim;
74 struct rs_codec *rs;
71 75
72 /* Allocate the control structure */ 76 rs = kzalloc(sizeof(*rs), gfp);
73 rs = kmalloc(sizeof (struct rs_control), GFP_KERNEL); 77 if (!rs)
74 if (rs == NULL)
75 return NULL; 78 return NULL;
76 79
77 INIT_LIST_HEAD(&rs->list); 80 INIT_LIST_HEAD(&rs->list);
@@ -85,17 +88,17 @@ static struct rs_control *rs_init(int symsize, int gfpoly, int (*gffunc)(int),
85 rs->gffunc = gffunc; 88 rs->gffunc = gffunc;
86 89
87 /* Allocate the arrays */ 90 /* Allocate the arrays */
88 rs->alpha_to = kmalloc(sizeof(uint16_t) * (rs->nn + 1), GFP_KERNEL); 91 rs->alpha_to = kmalloc_array(rs->nn + 1, sizeof(uint16_t), gfp);
89 if (rs->alpha_to == NULL) 92 if (rs->alpha_to == NULL)
90 goto errrs; 93 goto err;
91 94
92 rs->index_of = kmalloc(sizeof(uint16_t) * (rs->nn + 1), GFP_KERNEL); 95 rs->index_of = kmalloc_array(rs->nn + 1, sizeof(uint16_t), gfp);
93 if (rs->index_of == NULL) 96 if (rs->index_of == NULL)
94 goto erralp; 97 goto err;
95 98
96 rs->genpoly = kmalloc(sizeof(uint16_t) * (rs->nroots + 1), GFP_KERNEL); 99 rs->genpoly = kmalloc_array(rs->nroots + 1, sizeof(uint16_t), gfp);
97 if(rs->genpoly == NULL) 100 if(rs->genpoly == NULL)
98 goto erridx; 101 goto err;
99 102
100 /* Generate Galois field lookup tables */ 103 /* Generate Galois field lookup tables */
101 rs->index_of[0] = rs->nn; /* log(zero) = -inf */ 104 rs->index_of[0] = rs->nn; /* log(zero) = -inf */
@@ -120,7 +123,7 @@ static struct rs_control *rs_init(int symsize, int gfpoly, int (*gffunc)(int),
120 } 123 }
121 /* If it's not primitive, exit */ 124 /* If it's not primitive, exit */
122 if(sr != rs->alpha_to[0]) 125 if(sr != rs->alpha_to[0])
123 goto errpol; 126 goto err;
124 127
125 /* Find prim-th root of 1, used in decoding */ 128 /* Find prim-th root of 1, used in decoding */
126 for(iprim = 1; (iprim % prim) != 0; iprim += rs->nn); 129 for(iprim = 1; (iprim % prim) != 0; iprim += rs->nn);
@@ -148,42 +151,52 @@ static struct rs_control *rs_init(int symsize, int gfpoly, int (*gffunc)(int),
148 /* convert rs->genpoly[] to index form for quicker encoding */ 151 /* convert rs->genpoly[] to index form for quicker encoding */
149 for (i = 0; i <= nroots; i++) 152 for (i = 0; i <= nroots; i++)
150 rs->genpoly[i] = rs->index_of[rs->genpoly[i]]; 153 rs->genpoly[i] = rs->index_of[rs->genpoly[i]];
154
155 rs->users = 1;
156 list_add(&rs->list, &codec_list);
151 return rs; 157 return rs;
152 158
153 /* Error exit */ 159err:
154errpol:
155 kfree(rs->genpoly); 160 kfree(rs->genpoly);
156erridx:
157 kfree(rs->index_of); 161 kfree(rs->index_of);
158erralp:
159 kfree(rs->alpha_to); 162 kfree(rs->alpha_to);
160errrs:
161 kfree(rs); 163 kfree(rs);
162 return NULL; 164 return NULL;
163} 165}
164 166
165 167
166/** 168/**
167 * free_rs - Free the rs control structure, if it is no longer used 169 * free_rs - Free the rs control structure
168 * @rs: the control structure which is not longer used by the 170 * @rs: The control structure which is not longer used by the
169 * caller 171 * caller
172 *
173 * Free the control structure. If @rs is the last user of the associated
174 * codec, free the codec as well.
170 */ 175 */
171void free_rs(struct rs_control *rs) 176void free_rs(struct rs_control *rs)
172{ 177{
178 struct rs_codec *cd;
179
180 if (!rs)
181 return;
182
183 cd = rs->codec;
173 mutex_lock(&rslistlock); 184 mutex_lock(&rslistlock);
174 rs->users--; 185 cd->users--;
175 if(!rs->users) { 186 if(!cd->users) {
176 list_del(&rs->list); 187 list_del(&cd->list);
177 kfree(rs->alpha_to); 188 kfree(cd->alpha_to);
178 kfree(rs->index_of); 189 kfree(cd->index_of);
179 kfree(rs->genpoly); 190 kfree(cd->genpoly);
180 kfree(rs); 191 kfree(cd);
181 } 192 }
182 mutex_unlock(&rslistlock); 193 mutex_unlock(&rslistlock);
194 kfree(rs);
183} 195}
196EXPORT_SYMBOL_GPL(free_rs);
184 197
185/** 198/**
186 * init_rs_internal - Find a matching or allocate a new rs control structure 199 * init_rs_internal - Allocate rs control, find a matching codec or allocate a new one
187 * @symsize: the symbol size (number of bits) 200 * @symsize: the symbol size (number of bits)
188 * @gfpoly: the extended Galois field generator polynomial coefficients, 201 * @gfpoly: the extended Galois field generator polynomial coefficients,
189 * with the 0th coefficient in the low order bit. The polynomial 202 * with the 0th coefficient in the low order bit. The polynomial
@@ -191,55 +204,69 @@ void free_rs(struct rs_control *rs)
191 * @gffunc: pointer to function to generate the next field element, 204 * @gffunc: pointer to function to generate the next field element,
192 * or the multiplicative identity element if given 0. Used 205 * or the multiplicative identity element if given 0. Used
193 * instead of gfpoly if gfpoly is 0 206 * instead of gfpoly if gfpoly is 0
194 * @fcr: the first consecutive root of the rs code generator polynomial 207 * @fcr: the first consecutive root of the rs code generator polynomial
195 * in index form 208 * in index form
196 * @prim: primitive element to generate polynomial roots 209 * @prim: primitive element to generate polynomial roots
197 * @nroots: RS code generator polynomial degree (number of roots) 210 * @nroots: RS code generator polynomial degree (number of roots)
211 * @gfp: GFP_ flags for allocations
198 */ 212 */
199static struct rs_control *init_rs_internal(int symsize, int gfpoly, 213static struct rs_control *init_rs_internal(int symsize, int gfpoly,
200 int (*gffunc)(int), int fcr, 214 int (*gffunc)(int), int fcr,
201 int prim, int nroots) 215 int prim, int nroots, gfp_t gfp)
202{ 216{
203 struct list_head *tmp; 217 struct list_head *tmp;
204 struct rs_control *rs; 218 struct rs_control *rs;
219 unsigned int bsize;
205 220
206 /* Sanity checks */ 221 /* Sanity checks */
207 if (symsize < 1) 222 if (symsize < 1)
208 return NULL; 223 return NULL;
209 if (fcr < 0 || fcr >= (1<<symsize)) 224 if (fcr < 0 || fcr >= (1<<symsize))
210 return NULL; 225 return NULL;
211 if (prim <= 0 || prim >= (1<<symsize)) 226 if (prim <= 0 || prim >= (1<<symsize))
212 return NULL; 227 return NULL;
213 if (nroots < 0 || nroots >= (1<<symsize)) 228 if (nroots < 0 || nroots >= (1<<symsize))
214 return NULL; 229 return NULL;
215 230
231 /*
232 * The decoder needs buffers in each control struct instance to
233 * avoid variable size or large fixed size allocations on
234 * stack. Size the buffers to arrays of [nroots + 1].
235 */
236 bsize = sizeof(uint16_t) * RS_DECODE_NUM_BUFFERS * (nroots + 1);
237 rs = kzalloc(sizeof(*rs) + bsize, gfp);
238 if (!rs)
239 return NULL;
240
216 mutex_lock(&rslistlock); 241 mutex_lock(&rslistlock);
217 242
218 /* Walk through the list and look for a matching entry */ 243 /* Walk through the list and look for a matching entry */
219 list_for_each(tmp, &rslist) { 244 list_for_each(tmp, &codec_list) {
220 rs = list_entry(tmp, struct rs_control, list); 245 struct rs_codec *cd = list_entry(tmp, struct rs_codec, list);
221 if (symsize != rs->mm) 246
247 if (symsize != cd->mm)
222 continue; 248 continue;
223 if (gfpoly != rs->gfpoly) 249 if (gfpoly != cd->gfpoly)
224 continue; 250 continue;
225 if (gffunc != rs->gffunc) 251 if (gffunc != cd->gffunc)
226 continue; 252 continue;
227 if (fcr != rs->fcr) 253 if (fcr != cd->fcr)
228 continue; 254 continue;
229 if (prim != rs->prim) 255 if (prim != cd->prim)
230 continue; 256 continue;
231 if (nroots != rs->nroots) 257 if (nroots != cd->nroots)
232 continue; 258 continue;
233 /* We have a matching one already */ 259 /* We have a matching one already */
234 rs->users++; 260 cd->users++;
261 rs->codec = cd;
235 goto out; 262 goto out;
236 } 263 }
237 264
238 /* Create a new one */ 265 /* Create a new one */
239 rs = rs_init(symsize, gfpoly, gffunc, fcr, prim, nroots); 266 rs->codec = codec_init(symsize, gfpoly, gffunc, fcr, prim, nroots, gfp);
240 if (rs) { 267 if (!rs->codec) {
241 rs->users = 1; 268 kfree(rs);
242 list_add(&rs->list, &rslist); 269 rs = NULL;
243 } 270 }
244out: 271out:
245 mutex_unlock(&rslistlock); 272 mutex_unlock(&rslistlock);
@@ -247,45 +274,48 @@ out:
247} 274}
248 275
249/** 276/**
250 * init_rs - Find a matching or allocate a new rs control structure 277 * init_rs_gfp - Create a RS control struct and initialize it
251 * @symsize: the symbol size (number of bits) 278 * @symsize: the symbol size (number of bits)
252 * @gfpoly: the extended Galois field generator polynomial coefficients, 279 * @gfpoly: the extended Galois field generator polynomial coefficients,
253 * with the 0th coefficient in the low order bit. The polynomial 280 * with the 0th coefficient in the low order bit. The polynomial
254 * must be primitive; 281 * must be primitive;
255 * @fcr: the first consecutive root of the rs code generator polynomial 282 * @fcr: the first consecutive root of the rs code generator polynomial
256 * in index form 283 * in index form
257 * @prim: primitive element to generate polynomial roots 284 * @prim: primitive element to generate polynomial roots
258 * @nroots: RS code generator polynomial degree (number of roots) 285 * @nroots: RS code generator polynomial degree (number of roots)
286 * @gfp: GFP_ flags for allocations
259 */ 287 */
260struct rs_control *init_rs(int symsize, int gfpoly, int fcr, int prim, 288struct rs_control *init_rs_gfp(int symsize, int gfpoly, int fcr, int prim,
261 int nroots) 289 int nroots, gfp_t gfp)
262{ 290{
263 return init_rs_internal(symsize, gfpoly, NULL, fcr, prim, nroots); 291 return init_rs_internal(symsize, gfpoly, NULL, fcr, prim, nroots, gfp);
264} 292}
293EXPORT_SYMBOL_GPL(init_rs_gfp);
265 294
266/** 295/**
267 * init_rs_non_canonical - Find a matching or allocate a new rs control 296 * init_rs_non_canonical - Allocate rs control struct for fields with
268 * structure, for fields with non-canonical 297 * non-canonical representation
269 * representation
270 * @symsize: the symbol size (number of bits) 298 * @symsize: the symbol size (number of bits)
271 * @gffunc: pointer to function to generate the next field element, 299 * @gffunc: pointer to function to generate the next field element,
272 * or the multiplicative identity element if given 0. Used 300 * or the multiplicative identity element if given 0. Used
273 * instead of gfpoly if gfpoly is 0 301 * instead of gfpoly if gfpoly is 0
274 * @fcr: the first consecutive root of the rs code generator polynomial 302 * @fcr: the first consecutive root of the rs code generator polynomial
275 * in index form 303 * in index form
276 * @prim: primitive element to generate polynomial roots 304 * @prim: primitive element to generate polynomial roots
277 * @nroots: RS code generator polynomial degree (number of roots) 305 * @nroots: RS code generator polynomial degree (number of roots)
278 */ 306 */
279struct rs_control *init_rs_non_canonical(int symsize, int (*gffunc)(int), 307struct rs_control *init_rs_non_canonical(int symsize, int (*gffunc)(int),
280 int fcr, int prim, int nroots) 308 int fcr, int prim, int nroots)
281{ 309{
282 return init_rs_internal(symsize, 0, gffunc, fcr, prim, nroots); 310 return init_rs_internal(symsize, 0, gffunc, fcr, prim, nroots,
311 GFP_KERNEL);
283} 312}
313EXPORT_SYMBOL_GPL(init_rs_non_canonical);
284 314
285#ifdef CONFIG_REED_SOLOMON_ENC8 315#ifdef CONFIG_REED_SOLOMON_ENC8
286/** 316/**
287 * encode_rs8 - Calculate the parity for data values (8bit data width) 317 * encode_rs8 - Calculate the parity for data values (8bit data width)
288 * @rs: the rs control structure 318 * @rsc: the rs control structure
289 * @data: data field of a given type 319 * @data: data field of a given type
290 * @len: data length 320 * @len: data length
291 * @par: parity data, must be initialized by caller (usually all 0) 321 * @par: parity data, must be initialized by caller (usually all 0)
@@ -295,7 +325,7 @@ struct rs_control *init_rs_non_canonical(int symsize, int (*gffunc)(int),
295 * symbol size > 8. The calling code must take care of encoding of the 325 * symbol size > 8. The calling code must take care of encoding of the
296 * syndrome result for storage itself. 326 * syndrome result for storage itself.
297 */ 327 */
298int encode_rs8(struct rs_control *rs, uint8_t *data, int len, uint16_t *par, 328int encode_rs8(struct rs_control *rsc, uint8_t *data, int len, uint16_t *par,
299 uint16_t invmsk) 329 uint16_t invmsk)
300{ 330{
301#include "encode_rs.c" 331#include "encode_rs.c"
@@ -306,7 +336,7 @@ EXPORT_SYMBOL_GPL(encode_rs8);
306#ifdef CONFIG_REED_SOLOMON_DEC8 336#ifdef CONFIG_REED_SOLOMON_DEC8
307/** 337/**
308 * decode_rs8 - Decode codeword (8bit data width) 338 * decode_rs8 - Decode codeword (8bit data width)
309 * @rs: the rs control structure 339 * @rsc: the rs control structure
310 * @data: data field of a given type 340 * @data: data field of a given type
311 * @par: received parity data field 341 * @par: received parity data field
312 * @len: data length 342 * @len: data length
@@ -319,9 +349,14 @@ EXPORT_SYMBOL_GPL(encode_rs8);
319 * The syndrome and parity uses a uint16_t data type to enable 349 * The syndrome and parity uses a uint16_t data type to enable
320 * symbol size > 8. The calling code must take care of decoding of the 350 * symbol size > 8. The calling code must take care of decoding of the
321 * syndrome result and the received parity before calling this code. 351 * syndrome result and the received parity before calling this code.
352 *
353 * Note: The rs_control struct @rsc contains buffers which are used for
354 * decoding, so the caller has to ensure that decoder invocations are
355 * serialized.
356 *
322 * Returns the number of corrected bits or -EBADMSG for uncorrectable errors. 357 * Returns the number of corrected bits or -EBADMSG for uncorrectable errors.
323 */ 358 */
324int decode_rs8(struct rs_control *rs, uint8_t *data, uint16_t *par, int len, 359int decode_rs8(struct rs_control *rsc, uint8_t *data, uint16_t *par, int len,
325 uint16_t *s, int no_eras, int *eras_pos, uint16_t invmsk, 360 uint16_t *s, int no_eras, int *eras_pos, uint16_t invmsk,
326 uint16_t *corr) 361 uint16_t *corr)
327{ 362{
@@ -333,7 +368,7 @@ EXPORT_SYMBOL_GPL(decode_rs8);
333#ifdef CONFIG_REED_SOLOMON_ENC16 368#ifdef CONFIG_REED_SOLOMON_ENC16
334/** 369/**
335 * encode_rs16 - Calculate the parity for data values (16bit data width) 370 * encode_rs16 - Calculate the parity for data values (16bit data width)
336 * @rs: the rs control structure 371 * @rsc: the rs control structure
337 * @data: data field of a given type 372 * @data: data field of a given type
338 * @len: data length 373 * @len: data length
339 * @par: parity data, must be initialized by caller (usually all 0) 374 * @par: parity data, must be initialized by caller (usually all 0)
@@ -341,7 +376,7 @@ EXPORT_SYMBOL_GPL(decode_rs8);
341 * 376 *
342 * Each field in the data array contains up to symbol size bits of valid data. 377 * Each field in the data array contains up to symbol size bits of valid data.
343 */ 378 */
344int encode_rs16(struct rs_control *rs, uint16_t *data, int len, uint16_t *par, 379int encode_rs16(struct rs_control *rsc, uint16_t *data, int len, uint16_t *par,
345 uint16_t invmsk) 380 uint16_t invmsk)
346{ 381{
347#include "encode_rs.c" 382#include "encode_rs.c"
@@ -352,7 +387,7 @@ EXPORT_SYMBOL_GPL(encode_rs16);
352#ifdef CONFIG_REED_SOLOMON_DEC16 387#ifdef CONFIG_REED_SOLOMON_DEC16
353/** 388/**
354 * decode_rs16 - Decode codeword (16bit data width) 389 * decode_rs16 - Decode codeword (16bit data width)
355 * @rs: the rs control structure 390 * @rsc: the rs control structure
356 * @data: data field of a given type 391 * @data: data field of a given type
357 * @par: received parity data field 392 * @par: received parity data field
358 * @len: data length 393 * @len: data length
@@ -363,9 +398,14 @@ EXPORT_SYMBOL_GPL(encode_rs16);
363 * @corr: buffer to store correction bitmask on eras_pos 398 * @corr: buffer to store correction bitmask on eras_pos
364 * 399 *
365 * Each field in the data array contains up to symbol size bits of valid data. 400 * Each field in the data array contains up to symbol size bits of valid data.
401 *
402 * Note: The rc_control struct @rsc contains buffers which are used for
403 * decoding, so the caller has to ensure that decoder invocations are
404 * serialized.
405 *
366 * Returns the number of corrected bits or -EBADMSG for uncorrectable errors. 406 * Returns the number of corrected bits or -EBADMSG for uncorrectable errors.
367 */ 407 */
368int decode_rs16(struct rs_control *rs, uint16_t *data, uint16_t *par, int len, 408int decode_rs16(struct rs_control *rsc, uint16_t *data, uint16_t *par, int len,
369 uint16_t *s, int no_eras, int *eras_pos, uint16_t invmsk, 409 uint16_t *s, int no_eras, int *eras_pos, uint16_t invmsk,
370 uint16_t *corr) 410 uint16_t *corr)
371{ 411{
@@ -374,10 +414,6 @@ int decode_rs16(struct rs_control *rs, uint16_t *data, uint16_t *par, int len,
374EXPORT_SYMBOL_GPL(decode_rs16); 414EXPORT_SYMBOL_GPL(decode_rs16);
375#endif 415#endif
376 416
377EXPORT_SYMBOL_GPL(init_rs);
378EXPORT_SYMBOL_GPL(init_rs_non_canonical);
379EXPORT_SYMBOL_GPL(free_rs);
380
381MODULE_LICENSE("GPL"); 417MODULE_LICENSE("GPL");
382MODULE_DESCRIPTION("Reed Solomon encoder/decoder"); 418MODULE_DESCRIPTION("Reed Solomon encoder/decoder");
383MODULE_AUTHOR("Phil Karn, Thomas Gleixner"); 419MODULE_AUTHOR("Phil Karn, Thomas Gleixner");
diff --git a/lib/refcount.c b/lib/refcount.c
index 0eb48353abe3..d3b81cefce91 100644
--- a/lib/refcount.c
+++ b/lib/refcount.c
@@ -350,3 +350,31 @@ bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock)
350} 350}
351EXPORT_SYMBOL(refcount_dec_and_lock); 351EXPORT_SYMBOL(refcount_dec_and_lock);
352 352
353/**
354 * refcount_dec_and_lock_irqsave - return holding spinlock with disabled
355 * interrupts if able to decrement refcount to 0
356 * @r: the refcount
357 * @lock: the spinlock to be locked
358 * @flags: saved IRQ-flags if the is acquired
359 *
360 * Same as refcount_dec_and_lock() above except that the spinlock is acquired
361 * with disabled interupts.
362 *
363 * Return: true and hold spinlock if able to decrement refcount to 0, false
364 * otherwise
365 */
366bool refcount_dec_and_lock_irqsave(refcount_t *r, spinlock_t *lock,
367 unsigned long *flags)
368{
369 if (refcount_dec_not_one(r))
370 return false;
371
372 spin_lock_irqsave(lock, *flags);
373 if (!refcount_dec_and_test(r)) {
374 spin_unlock_irqrestore(lock, *flags);
375 return false;
376 }
377
378 return true;
379}
380EXPORT_SYMBOL(refcount_dec_and_lock_irqsave);
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 2b2b79974b61..9427b5766134 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -668,8 +668,9 @@ EXPORT_SYMBOL_GPL(rhashtable_insert_slow);
668 * For a completely stable walk you should construct your own data 668 * For a completely stable walk you should construct your own data
669 * structure outside the hash table. 669 * structure outside the hash table.
670 * 670 *
671 * This function may sleep so you must not call it from interrupt 671 * This function may be called from any process context, including
672 * context or with spin locks held. 672 * non-preemptable context, but cannot be called from softirq or
673 * hardirq context.
673 * 674 *
674 * You must call rhashtable_walk_exit after this function returns. 675 * You must call rhashtable_walk_exit after this function returns.
675 */ 676 */
@@ -726,6 +727,7 @@ int rhashtable_walk_start_check(struct rhashtable_iter *iter)
726 __acquires(RCU) 727 __acquires(RCU)
727{ 728{
728 struct rhashtable *ht = iter->ht; 729 struct rhashtable *ht = iter->ht;
730 bool rhlist = ht->rhlist;
729 731
730 rcu_read_lock(); 732 rcu_read_lock();
731 733
@@ -734,11 +736,52 @@ int rhashtable_walk_start_check(struct rhashtable_iter *iter)
734 list_del(&iter->walker.list); 736 list_del(&iter->walker.list);
735 spin_unlock(&ht->lock); 737 spin_unlock(&ht->lock);
736 738
737 if (!iter->walker.tbl && !iter->end_of_table) { 739 if (iter->end_of_table)
740 return 0;
741 if (!iter->walker.tbl) {
738 iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht); 742 iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht);
743 iter->slot = 0;
744 iter->skip = 0;
739 return -EAGAIN; 745 return -EAGAIN;
740 } 746 }
741 747
748 if (iter->p && !rhlist) {
749 /*
750 * We need to validate that 'p' is still in the table, and
751 * if so, update 'skip'
752 */
753 struct rhash_head *p;
754 int skip = 0;
755 rht_for_each_rcu(p, iter->walker.tbl, iter->slot) {
756 skip++;
757 if (p == iter->p) {
758 iter->skip = skip;
759 goto found;
760 }
761 }
762 iter->p = NULL;
763 } else if (iter->p && rhlist) {
764 /* Need to validate that 'list' is still in the table, and
765 * if so, update 'skip' and 'p'.
766 */
767 struct rhash_head *p;
768 struct rhlist_head *list;
769 int skip = 0;
770 rht_for_each_rcu(p, iter->walker.tbl, iter->slot) {
771 for (list = container_of(p, struct rhlist_head, rhead);
772 list;
773 list = rcu_dereference(list->next)) {
774 skip++;
775 if (list == iter->list) {
776 iter->p = p;
777 skip = skip;
778 goto found;
779 }
780 }
781 }
782 iter->p = NULL;
783 }
784found:
742 return 0; 785 return 0;
743} 786}
744EXPORT_SYMBOL_GPL(rhashtable_walk_start_check); 787EXPORT_SYMBOL_GPL(rhashtable_walk_start_check);
@@ -914,8 +957,6 @@ void rhashtable_walk_stop(struct rhashtable_iter *iter)
914 iter->walker.tbl = NULL; 957 iter->walker.tbl = NULL;
915 spin_unlock(&ht->lock); 958 spin_unlock(&ht->lock);
916 959
917 iter->p = NULL;
918
919out: 960out:
920 rcu_read_unlock(); 961 rcu_read_unlock();
921} 962}
diff --git a/lib/sbitmap.c b/lib/sbitmap.c
index e6a9c06ec70c..fdd1b8aa8ac6 100644
--- a/lib/sbitmap.c
+++ b/lib/sbitmap.c
@@ -52,7 +52,7 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
52 return 0; 52 return 0;
53 } 53 }
54 54
55 sb->map = kzalloc_node(sb->map_nr * sizeof(*sb->map), flags, node); 55 sb->map = kcalloc_node(sb->map_nr, sizeof(*sb->map), flags, node);
56 if (!sb->map) 56 if (!sb->map)
57 return -ENOMEM; 57 return -ENOMEM;
58 58
@@ -270,18 +270,33 @@ void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m)
270} 270}
271EXPORT_SYMBOL_GPL(sbitmap_bitmap_show); 271EXPORT_SYMBOL_GPL(sbitmap_bitmap_show);
272 272
273static unsigned int sbq_calc_wake_batch(unsigned int depth) 273static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq,
274 unsigned int depth)
274{ 275{
275 unsigned int wake_batch; 276 unsigned int wake_batch;
277 unsigned int shallow_depth;
276 278
277 /* 279 /*
278 * For each batch, we wake up one queue. We need to make sure that our 280 * For each batch, we wake up one queue. We need to make sure that our
279 * batch size is small enough that the full depth of the bitmap is 281 * batch size is small enough that the full depth of the bitmap,
280 * enough to wake up all of the queues. 282 * potentially limited by a shallow depth, is enough to wake up all of
283 * the queues.
284 *
285 * Each full word of the bitmap has bits_per_word bits, and there might
286 * be a partial word. There are depth / bits_per_word full words and
287 * depth % bits_per_word bits left over. In bitwise arithmetic:
288 *
289 * bits_per_word = 1 << shift
290 * depth / bits_per_word = depth >> shift
291 * depth % bits_per_word = depth & ((1 << shift) - 1)
292 *
293 * Each word can be limited to sbq->min_shallow_depth bits.
281 */ 294 */
282 wake_batch = SBQ_WAKE_BATCH; 295 shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth);
283 if (wake_batch > depth / SBQ_WAIT_QUEUES) 296 depth = ((depth >> sbq->sb.shift) * shallow_depth +
284 wake_batch = max(1U, depth / SBQ_WAIT_QUEUES); 297 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth));
298 wake_batch = clamp_t(unsigned int, depth / SBQ_WAIT_QUEUES, 1,
299 SBQ_WAKE_BATCH);
285 300
286 return wake_batch; 301 return wake_batch;
287} 302}
@@ -307,7 +322,8 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
307 *per_cpu_ptr(sbq->alloc_hint, i) = prandom_u32() % depth; 322 *per_cpu_ptr(sbq->alloc_hint, i) = prandom_u32() % depth;
308 } 323 }
309 324
310 sbq->wake_batch = sbq_calc_wake_batch(depth); 325 sbq->min_shallow_depth = UINT_MAX;
326 sbq->wake_batch = sbq_calc_wake_batch(sbq, depth);
311 atomic_set(&sbq->wake_index, 0); 327 atomic_set(&sbq->wake_index, 0);
312 328
313 sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node); 329 sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node);
@@ -327,21 +343,28 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth,
327} 343}
328EXPORT_SYMBOL_GPL(sbitmap_queue_init_node); 344EXPORT_SYMBOL_GPL(sbitmap_queue_init_node);
329 345
330void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth) 346static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq,
347 unsigned int depth)
331{ 348{
332 unsigned int wake_batch = sbq_calc_wake_batch(depth); 349 unsigned int wake_batch = sbq_calc_wake_batch(sbq, depth);
333 int i; 350 int i;
334 351
335 if (sbq->wake_batch != wake_batch) { 352 if (sbq->wake_batch != wake_batch) {
336 WRITE_ONCE(sbq->wake_batch, wake_batch); 353 WRITE_ONCE(sbq->wake_batch, wake_batch);
337 /* 354 /*
338 * Pairs with the memory barrier in sbq_wake_up() to ensure that 355 * Pairs with the memory barrier in sbitmap_queue_wake_up()
339 * the batch size is updated before the wait counts. 356 * to ensure that the batch size is updated before the wait
357 * counts.
340 */ 358 */
341 smp_mb__before_atomic(); 359 smp_mb__before_atomic();
342 for (i = 0; i < SBQ_WAIT_QUEUES; i++) 360 for (i = 0; i < SBQ_WAIT_QUEUES; i++)
343 atomic_set(&sbq->ws[i].wait_cnt, 1); 361 atomic_set(&sbq->ws[i].wait_cnt, 1);
344 } 362 }
363}
364
365void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth)
366{
367 sbitmap_queue_update_wake_batch(sbq, depth);
345 sbitmap_resize(&sbq->sb, depth); 368 sbitmap_resize(&sbq->sb, depth);
346} 369}
347EXPORT_SYMBOL_GPL(sbitmap_queue_resize); 370EXPORT_SYMBOL_GPL(sbitmap_queue_resize);
@@ -380,6 +403,8 @@ int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
380 unsigned int hint, depth; 403 unsigned int hint, depth;
381 int nr; 404 int nr;
382 405
406 WARN_ON_ONCE(shallow_depth < sbq->min_shallow_depth);
407
383 hint = this_cpu_read(*sbq->alloc_hint); 408 hint = this_cpu_read(*sbq->alloc_hint);
384 depth = READ_ONCE(sbq->sb.depth); 409 depth = READ_ONCE(sbq->sb.depth);
385 if (unlikely(hint >= depth)) { 410 if (unlikely(hint >= depth)) {
@@ -403,6 +428,14 @@ int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
403} 428}
404EXPORT_SYMBOL_GPL(__sbitmap_queue_get_shallow); 429EXPORT_SYMBOL_GPL(__sbitmap_queue_get_shallow);
405 430
431void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq,
432 unsigned int min_shallow_depth)
433{
434 sbq->min_shallow_depth = min_shallow_depth;
435 sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth);
436}
437EXPORT_SYMBOL_GPL(sbitmap_queue_min_shallow_depth);
438
406static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq) 439static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq)
407{ 440{
408 int i, wake_index; 441 int i, wake_index;
@@ -425,52 +458,67 @@ static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq)
425 return NULL; 458 return NULL;
426} 459}
427 460
428static void sbq_wake_up(struct sbitmap_queue *sbq) 461static bool __sbq_wake_up(struct sbitmap_queue *sbq)
429{ 462{
430 struct sbq_wait_state *ws; 463 struct sbq_wait_state *ws;
431 unsigned int wake_batch; 464 unsigned int wake_batch;
432 int wait_cnt; 465 int wait_cnt;
433 466
434 /*
435 * Pairs with the memory barrier in set_current_state() to ensure the
436 * proper ordering of clear_bit()/waitqueue_active() in the waker and
437 * test_and_set_bit_lock()/prepare_to_wait()/finish_wait() in the
438 * waiter. See the comment on waitqueue_active(). This is __after_atomic
439 * because we just did clear_bit_unlock() in the caller.
440 */
441 smp_mb__after_atomic();
442
443 ws = sbq_wake_ptr(sbq); 467 ws = sbq_wake_ptr(sbq);
444 if (!ws) 468 if (!ws)
445 return; 469 return false;
446 470
447 wait_cnt = atomic_dec_return(&ws->wait_cnt); 471 wait_cnt = atomic_dec_return(&ws->wait_cnt);
448 if (wait_cnt <= 0) { 472 if (wait_cnt <= 0) {
473 int ret;
474
449 wake_batch = READ_ONCE(sbq->wake_batch); 475 wake_batch = READ_ONCE(sbq->wake_batch);
476
450 /* 477 /*
451 * Pairs with the memory barrier in sbitmap_queue_resize() to 478 * Pairs with the memory barrier in sbitmap_queue_resize() to
452 * ensure that we see the batch size update before the wait 479 * ensure that we see the batch size update before the wait
453 * count is reset. 480 * count is reset.
454 */ 481 */
455 smp_mb__before_atomic(); 482 smp_mb__before_atomic();
483
456 /* 484 /*
457 * If there are concurrent callers to sbq_wake_up(), the last 485 * For concurrent callers of this, the one that failed the
458 * one to decrement the wait count below zero will bump it back 486 * atomic_cmpxhcg() race should call this function again
459 * up. If there is a concurrent resize, the count reset will 487 * to wakeup a new batch on a different 'ws'.
460 * either cause the cmpxchg to fail or overwrite after the
461 * cmpxchg.
462 */ 488 */
463 atomic_cmpxchg(&ws->wait_cnt, wait_cnt, wait_cnt + wake_batch); 489 ret = atomic_cmpxchg(&ws->wait_cnt, wait_cnt, wake_batch);
464 sbq_index_atomic_inc(&sbq->wake_index); 490 if (ret == wait_cnt) {
465 wake_up_nr(&ws->wait, wake_batch); 491 sbq_index_atomic_inc(&sbq->wake_index);
492 wake_up_nr(&ws->wait, wake_batch);
493 return false;
494 }
495
496 return true;
466 } 497 }
498
499 return false;
500}
501
502void sbitmap_queue_wake_up(struct sbitmap_queue *sbq)
503{
504 while (__sbq_wake_up(sbq))
505 ;
467} 506}
507EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up);
468 508
469void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, 509void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr,
470 unsigned int cpu) 510 unsigned int cpu)
471{ 511{
472 sbitmap_clear_bit_unlock(&sbq->sb, nr); 512 sbitmap_clear_bit_unlock(&sbq->sb, nr);
473 sbq_wake_up(sbq); 513 /*
514 * Pairs with the memory barrier in set_current_state() to ensure the
515 * proper ordering of clear_bit_unlock()/waitqueue_active() in the waker
516 * and test_and_set_bit_lock()/prepare_to_wait()/finish_wait() in the
517 * waiter. See the comment on waitqueue_active().
518 */
519 smp_mb__after_atomic();
520 sbitmap_queue_wake_up(sbq);
521
474 if (likely(!sbq->round_robin && nr < sbq->sb.depth)) 522 if (likely(!sbq->round_robin && nr < sbq->sb.depth))
475 *per_cpu_ptr(sbq->alloc_hint, cpu) = nr; 523 *per_cpu_ptr(sbq->alloc_hint, cpu) = nr;
476} 524}
@@ -482,7 +530,7 @@ void sbitmap_queue_wake_all(struct sbitmap_queue *sbq)
482 530
483 /* 531 /*
484 * Pairs with the memory barrier in set_current_state() like in 532 * Pairs with the memory barrier in set_current_state() like in
485 * sbq_wake_up(). 533 * sbitmap_queue_wake_up().
486 */ 534 */
487 smp_mb(); 535 smp_mb();
488 wake_index = atomic_read(&sbq->wake_index); 536 wake_index = atomic_read(&sbq->wake_index);
@@ -528,5 +576,6 @@ void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m)
528 seq_puts(m, "}\n"); 576 seq_puts(m, "}\n");
529 577
530 seq_printf(m, "round_robin=%d\n", sbq->round_robin); 578 seq_printf(m, "round_robin=%d\n", sbq->round_robin);
579 seq_printf(m, "min_shallow_depth=%u\n", sbq->min_shallow_depth);
531} 580}
532EXPORT_SYMBOL_GPL(sbitmap_queue_show); 581EXPORT_SYMBOL_GPL(sbitmap_queue_show);
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index 06dad7a072fd..7c6096a71704 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -24,9 +24,6 @@
24 **/ 24 **/
25struct scatterlist *sg_next(struct scatterlist *sg) 25struct scatterlist *sg_next(struct scatterlist *sg)
26{ 26{
27#ifdef CONFIG_DEBUG_SG
28 BUG_ON(sg->sg_magic != SG_MAGIC);
29#endif
30 if (sg_is_last(sg)) 27 if (sg_is_last(sg))
31 return NULL; 28 return NULL;
32 29
@@ -111,10 +108,7 @@ struct scatterlist *sg_last(struct scatterlist *sgl, unsigned int nents)
111 for_each_sg(sgl, sg, nents, i) 108 for_each_sg(sgl, sg, nents, i)
112 ret = sg; 109 ret = sg;
113 110
114#ifdef CONFIG_DEBUG_SG
115 BUG_ON(sgl[0].sg_magic != SG_MAGIC);
116 BUG_ON(!sg_is_last(ret)); 111 BUG_ON(!sg_is_last(ret));
117#endif
118 return ret; 112 return ret;
119} 113}
120EXPORT_SYMBOL(sg_last); 114EXPORT_SYMBOL(sg_last);
@@ -170,7 +164,8 @@ static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
170 kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask); 164 kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask);
171 return ptr; 165 return ptr;
172 } else 166 } else
173 return kmalloc(nents * sizeof(struct scatterlist), gfp_mask); 167 return kmalloc_array(nents, sizeof(struct scatterlist),
168 gfp_mask);
174} 169}
175 170
176static void sg_kfree(struct scatterlist *sg, unsigned int nents) 171static void sg_kfree(struct scatterlist *sg, unsigned int nents)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
deleted file mode 100644
index cc640588f145..000000000000
--- a/lib/swiotlb.c
+++ /dev/null
@@ -1,1092 +0,0 @@
1/*
2 * Dynamic DMA mapping support.
3 *
4 * This implementation is a fallback for platforms that do not support
5 * I/O TLBs (aka DMA address translation hardware).
6 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
7 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
8 * Copyright (C) 2000, 2003 Hewlett-Packard Co
9 * David Mosberger-Tang <davidm@hpl.hp.com>
10 *
11 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
12 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
13 * unnecessary i-cache flushing.
14 * 04/07/.. ak Better overflow handling. Assorted fixes.
15 * 05/09/10 linville Add support for syncing ranges, support syncing for
16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
17 * 08/12/11 beckyb Add highmem support
18 */
19
20#include <linux/cache.h>
21#include <linux/dma-direct.h>
22#include <linux/mm.h>
23#include <linux/export.h>
24#include <linux/spinlock.h>
25#include <linux/string.h>
26#include <linux/swiotlb.h>
27#include <linux/pfn.h>
28#include <linux/types.h>
29#include <linux/ctype.h>
30#include <linux/highmem.h>
31#include <linux/gfp.h>
32#include <linux/scatterlist.h>
33#include <linux/mem_encrypt.h>
34#include <linux/set_memory.h>
35
36#include <asm/io.h>
37#include <asm/dma.h>
38
39#include <linux/init.h>
40#include <linux/bootmem.h>
41#include <linux/iommu-helper.h>
42
43#define CREATE_TRACE_POINTS
44#include <trace/events/swiotlb.h>
45
46#define OFFSET(val,align) ((unsigned long) \
47 ( (val) & ( (align) - 1)))
48
49#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
50
51/*
52 * Minimum IO TLB size to bother booting with. Systems with mainly
53 * 64bit capable cards will only lightly use the swiotlb. If we can't
54 * allocate a contiguous 1MB, we're probably in trouble anyway.
55 */
56#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
57
58enum swiotlb_force swiotlb_force;
59
60/*
61 * Used to do a quick range check in swiotlb_tbl_unmap_single and
62 * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
63 * API.
64 */
65static phys_addr_t io_tlb_start, io_tlb_end;
66
67/*
68 * The number of IO TLB blocks (in groups of 64) between io_tlb_start and
69 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
70 */
71static unsigned long io_tlb_nslabs;
72
73/*
74 * When the IOMMU overflows we return a fallback buffer. This sets the size.
75 */
76static unsigned long io_tlb_overflow = 32*1024;
77
78static phys_addr_t io_tlb_overflow_buffer;
79
80/*
81 * This is a free list describing the number of free entries available from
82 * each index
83 */
84static unsigned int *io_tlb_list;
85static unsigned int io_tlb_index;
86
87/*
88 * Max segment that we can provide which (if pages are contingous) will
89 * not be bounced (unless SWIOTLB_FORCE is set).
90 */
91unsigned int max_segment;
92
93/*
94 * We need to save away the original address corresponding to a mapped entry
95 * for the sync operations.
96 */
97#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
98static phys_addr_t *io_tlb_orig_addr;
99
100/*
101 * Protect the above data structures in the map and unmap calls
102 */
103static DEFINE_SPINLOCK(io_tlb_lock);
104
105static int late_alloc;
106
107static int __init
108setup_io_tlb_npages(char *str)
109{
110 if (isdigit(*str)) {
111 io_tlb_nslabs = simple_strtoul(str, &str, 0);
112 /* avoid tail segment of size < IO_TLB_SEGSIZE */
113 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
114 }
115 if (*str == ',')
116 ++str;
117 if (!strcmp(str, "force")) {
118 swiotlb_force = SWIOTLB_FORCE;
119 } else if (!strcmp(str, "noforce")) {
120 swiotlb_force = SWIOTLB_NO_FORCE;
121 io_tlb_nslabs = 1;
122 }
123
124 return 0;
125}
126early_param("swiotlb", setup_io_tlb_npages);
127/* make io_tlb_overflow tunable too? */
128
129unsigned long swiotlb_nr_tbl(void)
130{
131 return io_tlb_nslabs;
132}
133EXPORT_SYMBOL_GPL(swiotlb_nr_tbl);
134
135unsigned int swiotlb_max_segment(void)
136{
137 return max_segment;
138}
139EXPORT_SYMBOL_GPL(swiotlb_max_segment);
140
141void swiotlb_set_max_segment(unsigned int val)
142{
143 if (swiotlb_force == SWIOTLB_FORCE)
144 max_segment = 1;
145 else
146 max_segment = rounddown(val, PAGE_SIZE);
147}
148
149/* default to 64MB */
150#define IO_TLB_DEFAULT_SIZE (64UL<<20)
151unsigned long swiotlb_size_or_default(void)
152{
153 unsigned long size;
154
155 size = io_tlb_nslabs << IO_TLB_SHIFT;
156
157 return size ? size : (IO_TLB_DEFAULT_SIZE);
158}
159
160static bool no_iotlb_memory;
161
162void swiotlb_print_info(void)
163{
164 unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT;
165 unsigned char *vstart, *vend;
166
167 if (no_iotlb_memory) {
168 pr_warn("software IO TLB: No low mem\n");
169 return;
170 }
171
172 vstart = phys_to_virt(io_tlb_start);
173 vend = phys_to_virt(io_tlb_end);
174
175 printk(KERN_INFO "software IO TLB [mem %#010llx-%#010llx] (%luMB) mapped at [%p-%p]\n",
176 (unsigned long long)io_tlb_start,
177 (unsigned long long)io_tlb_end,
178 bytes >> 20, vstart, vend - 1);
179}
180
181/*
182 * Early SWIOTLB allocation may be too early to allow an architecture to
183 * perform the desired operations. This function allows the architecture to
184 * call SWIOTLB when the operations are possible. It needs to be called
185 * before the SWIOTLB memory is used.
186 */
187void __init swiotlb_update_mem_attributes(void)
188{
189 void *vaddr;
190 unsigned long bytes;
191
192 if (no_iotlb_memory || late_alloc)
193 return;
194
195 vaddr = phys_to_virt(io_tlb_start);
196 bytes = PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT);
197 set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
198 memset(vaddr, 0, bytes);
199
200 vaddr = phys_to_virt(io_tlb_overflow_buffer);
201 bytes = PAGE_ALIGN(io_tlb_overflow);
202 set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
203 memset(vaddr, 0, bytes);
204}
205
206int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
207{
208 void *v_overflow_buffer;
209 unsigned long i, bytes;
210
211 bytes = nslabs << IO_TLB_SHIFT;
212
213 io_tlb_nslabs = nslabs;
214 io_tlb_start = __pa(tlb);
215 io_tlb_end = io_tlb_start + bytes;
216
217 /*
218 * Get the overflow emergency buffer
219 */
220 v_overflow_buffer = memblock_virt_alloc_low_nopanic(
221 PAGE_ALIGN(io_tlb_overflow),
222 PAGE_SIZE);
223 if (!v_overflow_buffer)
224 return -ENOMEM;
225
226 io_tlb_overflow_buffer = __pa(v_overflow_buffer);
227
228 /*
229 * Allocate and initialize the free list array. This array is used
230 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
231 * between io_tlb_start and io_tlb_end.
232 */
233 io_tlb_list = memblock_virt_alloc(
234 PAGE_ALIGN(io_tlb_nslabs * sizeof(int)),
235 PAGE_SIZE);
236 io_tlb_orig_addr = memblock_virt_alloc(
237 PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)),
238 PAGE_SIZE);
239 for (i = 0; i < io_tlb_nslabs; i++) {
240 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
241 io_tlb_orig_addr[i] = INVALID_PHYS_ADDR;
242 }
243 io_tlb_index = 0;
244
245 if (verbose)
246 swiotlb_print_info();
247
248 swiotlb_set_max_segment(io_tlb_nslabs << IO_TLB_SHIFT);
249 return 0;
250}
251
252/*
253 * Statically reserve bounce buffer space and initialize bounce buffer data
254 * structures for the software IO TLB used to implement the DMA API.
255 */
256void __init
257swiotlb_init(int verbose)
258{
259 size_t default_size = IO_TLB_DEFAULT_SIZE;
260 unsigned char *vstart;
261 unsigned long bytes;
262
263 if (!io_tlb_nslabs) {
264 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
265 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
266 }
267
268 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
269
270 /* Get IO TLB memory from the low pages */
271 vstart = memblock_virt_alloc_low_nopanic(PAGE_ALIGN(bytes), PAGE_SIZE);
272 if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose))
273 return;
274
275 if (io_tlb_start)
276 memblock_free_early(io_tlb_start,
277 PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));
278 pr_warn("Cannot allocate SWIOTLB buffer");
279 no_iotlb_memory = true;
280}
281
282/*
283 * Systems with larger DMA zones (those that don't support ISA) can
284 * initialize the swiotlb later using the slab allocator if needed.
285 * This should be just like above, but with some error catching.
286 */
287int
288swiotlb_late_init_with_default_size(size_t default_size)
289{
290 unsigned long bytes, req_nslabs = io_tlb_nslabs;
291 unsigned char *vstart = NULL;
292 unsigned int order;
293 int rc = 0;
294
295 if (!io_tlb_nslabs) {
296 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
297 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
298 }
299
300 /*
301 * Get IO TLB memory from the low pages
302 */
303 order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
304 io_tlb_nslabs = SLABS_PER_PAGE << order;
305 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
306
307 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
308 vstart = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
309 order);
310 if (vstart)
311 break;
312 order--;
313 }
314
315 if (!vstart) {
316 io_tlb_nslabs = req_nslabs;
317 return -ENOMEM;
318 }
319 if (order != get_order(bytes)) {
320 printk(KERN_WARNING "Warning: only able to allocate %ld MB "
321 "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
322 io_tlb_nslabs = SLABS_PER_PAGE << order;
323 }
324 rc = swiotlb_late_init_with_tbl(vstart, io_tlb_nslabs);
325 if (rc)
326 free_pages((unsigned long)vstart, order);
327
328 return rc;
329}
330
331int
332swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
333{
334 unsigned long i, bytes;
335 unsigned char *v_overflow_buffer;
336
337 bytes = nslabs << IO_TLB_SHIFT;
338
339 io_tlb_nslabs = nslabs;
340 io_tlb_start = virt_to_phys(tlb);
341 io_tlb_end = io_tlb_start + bytes;
342
343 set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
344 memset(tlb, 0, bytes);
345
346 /*
347 * Get the overflow emergency buffer
348 */
349 v_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
350 get_order(io_tlb_overflow));
351 if (!v_overflow_buffer)
352 goto cleanup2;
353
354 set_memory_decrypted((unsigned long)v_overflow_buffer,
355 io_tlb_overflow >> PAGE_SHIFT);
356 memset(v_overflow_buffer, 0, io_tlb_overflow);
357 io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer);
358
359 /*
360 * Allocate and initialize the free list array. This array is used
361 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
362 * between io_tlb_start and io_tlb_end.
363 */
364 io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
365 get_order(io_tlb_nslabs * sizeof(int)));
366 if (!io_tlb_list)
367 goto cleanup3;
368
369 io_tlb_orig_addr = (phys_addr_t *)
370 __get_free_pages(GFP_KERNEL,
371 get_order(io_tlb_nslabs *
372 sizeof(phys_addr_t)));
373 if (!io_tlb_orig_addr)
374 goto cleanup4;
375
376 for (i = 0; i < io_tlb_nslabs; i++) {
377 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
378 io_tlb_orig_addr[i] = INVALID_PHYS_ADDR;
379 }
380 io_tlb_index = 0;
381
382 swiotlb_print_info();
383
384 late_alloc = 1;
385
386 swiotlb_set_max_segment(io_tlb_nslabs << IO_TLB_SHIFT);
387
388 return 0;
389
390cleanup4:
391 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
392 sizeof(int)));
393 io_tlb_list = NULL;
394cleanup3:
395 free_pages((unsigned long)v_overflow_buffer,
396 get_order(io_tlb_overflow));
397 io_tlb_overflow_buffer = 0;
398cleanup2:
399 io_tlb_end = 0;
400 io_tlb_start = 0;
401 io_tlb_nslabs = 0;
402 max_segment = 0;
403 return -ENOMEM;
404}
405
406void __init swiotlb_exit(void)
407{
408 if (!io_tlb_orig_addr)
409 return;
410
411 if (late_alloc) {
412 free_pages((unsigned long)phys_to_virt(io_tlb_overflow_buffer),
413 get_order(io_tlb_overflow));
414 free_pages((unsigned long)io_tlb_orig_addr,
415 get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
416 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
417 sizeof(int)));
418 free_pages((unsigned long)phys_to_virt(io_tlb_start),
419 get_order(io_tlb_nslabs << IO_TLB_SHIFT));
420 } else {
421 memblock_free_late(io_tlb_overflow_buffer,
422 PAGE_ALIGN(io_tlb_overflow));
423 memblock_free_late(__pa(io_tlb_orig_addr),
424 PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
425 memblock_free_late(__pa(io_tlb_list),
426 PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));
427 memblock_free_late(io_tlb_start,
428 PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));
429 }
430 io_tlb_nslabs = 0;
431 max_segment = 0;
432}
433
434int is_swiotlb_buffer(phys_addr_t paddr)
435{
436 return paddr >= io_tlb_start && paddr < io_tlb_end;
437}
438
439/*
440 * Bounce: copy the swiotlb buffer back to the original dma location
441 */
442static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,
443 size_t size, enum dma_data_direction dir)
444{
445 unsigned long pfn = PFN_DOWN(orig_addr);
446 unsigned char *vaddr = phys_to_virt(tlb_addr);
447
448 if (PageHighMem(pfn_to_page(pfn))) {
449 /* The buffer does not have a mapping. Map it in and copy */
450 unsigned int offset = orig_addr & ~PAGE_MASK;
451 char *buffer;
452 unsigned int sz = 0;
453 unsigned long flags;
454
455 while (size) {
456 sz = min_t(size_t, PAGE_SIZE - offset, size);
457
458 local_irq_save(flags);
459 buffer = kmap_atomic(pfn_to_page(pfn));
460 if (dir == DMA_TO_DEVICE)
461 memcpy(vaddr, buffer + offset, sz);
462 else
463 memcpy(buffer + offset, vaddr, sz);
464 kunmap_atomic(buffer);
465 local_irq_restore(flags);
466
467 size -= sz;
468 pfn++;
469 vaddr += sz;
470 offset = 0;
471 }
472 } else if (dir == DMA_TO_DEVICE) {
473 memcpy(vaddr, phys_to_virt(orig_addr), size);
474 } else {
475 memcpy(phys_to_virt(orig_addr), vaddr, size);
476 }
477}
478
479phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
480 dma_addr_t tbl_dma_addr,
481 phys_addr_t orig_addr, size_t size,
482 enum dma_data_direction dir,
483 unsigned long attrs)
484{
485 unsigned long flags;
486 phys_addr_t tlb_addr;
487 unsigned int nslots, stride, index, wrap;
488 int i;
489 unsigned long mask;
490 unsigned long offset_slots;
491 unsigned long max_slots;
492
493 if (no_iotlb_memory)
494 panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
495
496 if (mem_encrypt_active())
497 pr_warn_once("%s is active and system is using DMA bounce buffers\n",
498 sme_active() ? "SME" : "SEV");
499
500 mask = dma_get_seg_boundary(hwdev);
501
502 tbl_dma_addr &= mask;
503
504 offset_slots = ALIGN(tbl_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
505
506 /*
507 * Carefully handle integer overflow which can occur when mask == ~0UL.
508 */
509 max_slots = mask + 1
510 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT
511 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
512
513 /*
514 * For mappings greater than or equal to a page, we limit the stride
515 * (and hence alignment) to a page size.
516 */
517 nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
518 if (size >= PAGE_SIZE)
519 stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
520 else
521 stride = 1;
522
523 BUG_ON(!nslots);
524
525 /*
526 * Find suitable number of IO TLB entries size that will fit this
527 * request and allocate a buffer from that IO TLB pool.
528 */
529 spin_lock_irqsave(&io_tlb_lock, flags);
530 index = ALIGN(io_tlb_index, stride);
531 if (index >= io_tlb_nslabs)
532 index = 0;
533 wrap = index;
534
535 do {
536 while (iommu_is_span_boundary(index, nslots, offset_slots,
537 max_slots)) {
538 index += stride;
539 if (index >= io_tlb_nslabs)
540 index = 0;
541 if (index == wrap)
542 goto not_found;
543 }
544
545 /*
546 * If we find a slot that indicates we have 'nslots' number of
547 * contiguous buffers, we allocate the buffers from that slot
548 * and mark the entries as '0' indicating unavailable.
549 */
550 if (io_tlb_list[index] >= nslots) {
551 int count = 0;
552
553 for (i = index; i < (int) (index + nslots); i++)
554 io_tlb_list[i] = 0;
555 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)
556 io_tlb_list[i] = ++count;
557 tlb_addr = io_tlb_start + (index << IO_TLB_SHIFT);
558
559 /*
560 * Update the indices to avoid searching in the next
561 * round.
562 */
563 io_tlb_index = ((index + nslots) < io_tlb_nslabs
564 ? (index + nslots) : 0);
565
566 goto found;
567 }
568 index += stride;
569 if (index >= io_tlb_nslabs)
570 index = 0;
571 } while (index != wrap);
572
573not_found:
574 spin_unlock_irqrestore(&io_tlb_lock, flags);
575 if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
576 dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes)\n", size);
577 return SWIOTLB_MAP_ERROR;
578found:
579 spin_unlock_irqrestore(&io_tlb_lock, flags);
580
581 /*
582 * Save away the mapping from the original address to the DMA address.
583 * This is needed when we sync the memory. Then we sync the buffer if
584 * needed.
585 */
586 for (i = 0; i < nslots; i++)
587 io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT);
588 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
589 (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
590 swiotlb_bounce(orig_addr, tlb_addr, size, DMA_TO_DEVICE);
591
592 return tlb_addr;
593}
594
595/*
596 * Allocates bounce buffer and returns its kernel virtual address.
597 */
598
599static phys_addr_t
600map_single(struct device *hwdev, phys_addr_t phys, size_t size,
601 enum dma_data_direction dir, unsigned long attrs)
602{
603 dma_addr_t start_dma_addr;
604
605 if (swiotlb_force == SWIOTLB_NO_FORCE) {
606 dev_warn_ratelimited(hwdev, "Cannot do DMA to address %pa\n",
607 &phys);
608 return SWIOTLB_MAP_ERROR;
609 }
610
611 start_dma_addr = __phys_to_dma(hwdev, io_tlb_start);
612 return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
613 dir, attrs);
614}
615
616/*
617 * dma_addr is the kernel virtual address of the bounce buffer to unmap.
618 */
619void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
620 size_t size, enum dma_data_direction dir,
621 unsigned long attrs)
622{
623 unsigned long flags;
624 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
625 int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT;
626 phys_addr_t orig_addr = io_tlb_orig_addr[index];
627
628 /*
629 * First, sync the memory before unmapping the entry
630 */
631 if (orig_addr != INVALID_PHYS_ADDR &&
632 !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
633 ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
634 swiotlb_bounce(orig_addr, tlb_addr, size, DMA_FROM_DEVICE);
635
636 /*
637 * Return the buffer to the free list by setting the corresponding
638 * entries to indicate the number of contiguous entries available.
639 * While returning the entries to the free list, we merge the entries
640 * with slots below and above the pool being returned.
641 */
642 spin_lock_irqsave(&io_tlb_lock, flags);
643 {
644 count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
645 io_tlb_list[index + nslots] : 0);
646 /*
647 * Step 1: return the slots to the free list, merging the
648 * slots with superceeding slots
649 */
650 for (i = index + nslots - 1; i >= index; i--) {
651 io_tlb_list[i] = ++count;
652 io_tlb_orig_addr[i] = INVALID_PHYS_ADDR;
653 }
654 /*
655 * Step 2: merge the returned slots with the preceding slots,
656 * if available (non zero)
657 */
658 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
659 io_tlb_list[i] = ++count;
660 }
661 spin_unlock_irqrestore(&io_tlb_lock, flags);
662}
663
664void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
665 size_t size, enum dma_data_direction dir,
666 enum dma_sync_target target)
667{
668 int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT;
669 phys_addr_t orig_addr = io_tlb_orig_addr[index];
670
671 if (orig_addr == INVALID_PHYS_ADDR)
672 return;
673 orig_addr += (unsigned long)tlb_addr & ((1 << IO_TLB_SHIFT) - 1);
674
675 switch (target) {
676 case SYNC_FOR_CPU:
677 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
678 swiotlb_bounce(orig_addr, tlb_addr,
679 size, DMA_FROM_DEVICE);
680 else
681 BUG_ON(dir != DMA_TO_DEVICE);
682 break;
683 case SYNC_FOR_DEVICE:
684 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
685 swiotlb_bounce(orig_addr, tlb_addr,
686 size, DMA_TO_DEVICE);
687 else
688 BUG_ON(dir != DMA_FROM_DEVICE);
689 break;
690 default:
691 BUG();
692 }
693}
694
695#ifdef CONFIG_DMA_DIRECT_OPS
696static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
697 size_t size)
698{
699 u64 mask = DMA_BIT_MASK(32);
700
701 if (dev && dev->coherent_dma_mask)
702 mask = dev->coherent_dma_mask;
703 return addr + size - 1 <= mask;
704}
705
706static void *
707swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
708 unsigned long attrs)
709{
710 phys_addr_t phys_addr;
711
712 if (swiotlb_force == SWIOTLB_NO_FORCE)
713 goto out_warn;
714
715 phys_addr = swiotlb_tbl_map_single(dev,
716 __phys_to_dma(dev, io_tlb_start),
717 0, size, DMA_FROM_DEVICE, attrs);
718 if (phys_addr == SWIOTLB_MAP_ERROR)
719 goto out_warn;
720
721 *dma_handle = __phys_to_dma(dev, phys_addr);
722 if (!dma_coherent_ok(dev, *dma_handle, size))
723 goto out_unmap;
724
725 memset(phys_to_virt(phys_addr), 0, size);
726 return phys_to_virt(phys_addr);
727
728out_unmap:
729 dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
730 (unsigned long long)(dev ? dev->coherent_dma_mask : 0),
731 (unsigned long long)*dma_handle);
732
733 /*
734 * DMA_TO_DEVICE to avoid memcpy in unmap_single.
735 * DMA_ATTR_SKIP_CPU_SYNC is optional.
736 */
737 swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
738 DMA_ATTR_SKIP_CPU_SYNC);
739out_warn:
740 if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {
741 dev_warn(dev,
742 "swiotlb: coherent allocation failed, size=%zu\n",
743 size);
744 dump_stack();
745 }
746 return NULL;
747}
748
749static bool swiotlb_free_buffer(struct device *dev, size_t size,
750 dma_addr_t dma_addr)
751{
752 phys_addr_t phys_addr = dma_to_phys(dev, dma_addr);
753
754 WARN_ON_ONCE(irqs_disabled());
755
756 if (!is_swiotlb_buffer(phys_addr))
757 return false;
758
759 /*
760 * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single.
761 * DMA_ATTR_SKIP_CPU_SYNC is optional.
762 */
763 swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
764 DMA_ATTR_SKIP_CPU_SYNC);
765 return true;
766}
767#endif
768
769static void
770swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir,
771 int do_panic)
772{
773 if (swiotlb_force == SWIOTLB_NO_FORCE)
774 return;
775
776 /*
777 * Ran out of IOMMU space for this operation. This is very bad.
778 * Unfortunately the drivers cannot handle this operation properly.
779 * unless they check for dma_mapping_error (most don't)
780 * When the mapping is small enough return a static buffer to limit
781 * the damage, or panic when the transfer is too big.
782 */
783 dev_err_ratelimited(dev, "DMA: Out of SW-IOMMU space for %zu bytes\n",
784 size);
785
786 if (size <= io_tlb_overflow || !do_panic)
787 return;
788
789 if (dir == DMA_BIDIRECTIONAL)
790 panic("DMA: Random memory could be DMA accessed\n");
791 if (dir == DMA_FROM_DEVICE)
792 panic("DMA: Random memory could be DMA written\n");
793 if (dir == DMA_TO_DEVICE)
794 panic("DMA: Random memory could be DMA read\n");
795}
796
797/*
798 * Map a single buffer of the indicated size for DMA in streaming mode. The
799 * physical address to use is returned.
800 *
801 * Once the device is given the dma address, the device owns this memory until
802 * either swiotlb_unmap_page or swiotlb_dma_sync_single is performed.
803 */
804dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
805 unsigned long offset, size_t size,
806 enum dma_data_direction dir,
807 unsigned long attrs)
808{
809 phys_addr_t map, phys = page_to_phys(page) + offset;
810 dma_addr_t dev_addr = phys_to_dma(dev, phys);
811
812 BUG_ON(dir == DMA_NONE);
813 /*
814 * If the address happens to be in the device's DMA window,
815 * we can safely return the device addr and not worry about bounce
816 * buffering it.
817 */
818 if (dma_capable(dev, dev_addr, size) && swiotlb_force != SWIOTLB_FORCE)
819 return dev_addr;
820
821 trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
822
823 /* Oh well, have to allocate and map a bounce buffer. */
824 map = map_single(dev, phys, size, dir, attrs);
825 if (map == SWIOTLB_MAP_ERROR) {
826 swiotlb_full(dev, size, dir, 1);
827 return __phys_to_dma(dev, io_tlb_overflow_buffer);
828 }
829
830 dev_addr = __phys_to_dma(dev, map);
831
832 /* Ensure that the address returned is DMA'ble */
833 if (dma_capable(dev, dev_addr, size))
834 return dev_addr;
835
836 attrs |= DMA_ATTR_SKIP_CPU_SYNC;
837 swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
838
839 return __phys_to_dma(dev, io_tlb_overflow_buffer);
840}
841
842/*
843 * Unmap a single streaming mode DMA translation. The dma_addr and size must
844 * match what was provided for in a previous swiotlb_map_page call. All
845 * other usages are undefined.
846 *
847 * After this call, reads by the cpu to the buffer are guaranteed to see
848 * whatever the device wrote there.
849 */
850static void unmap_single(struct device *hwdev, dma_addr_t dev_addr,
851 size_t size, enum dma_data_direction dir,
852 unsigned long attrs)
853{
854 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
855
856 BUG_ON(dir == DMA_NONE);
857
858 if (is_swiotlb_buffer(paddr)) {
859 swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
860 return;
861 }
862
863 if (dir != DMA_FROM_DEVICE)
864 return;
865
866 /*
867 * phys_to_virt doesn't work with hihgmem page but we could
868 * call dma_mark_clean() with hihgmem page here. However, we
869 * are fine since dma_mark_clean() is null on POWERPC. We can
870 * make dma_mark_clean() take a physical address if necessary.
871 */
872 dma_mark_clean(phys_to_virt(paddr), size);
873}
874
875void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
876 size_t size, enum dma_data_direction dir,
877 unsigned long attrs)
878{
879 unmap_single(hwdev, dev_addr, size, dir, attrs);
880}
881
882/*
883 * Make physical memory consistent for a single streaming mode DMA translation
884 * after a transfer.
885 *
886 * If you perform a swiotlb_map_page() but wish to interrogate the buffer
887 * using the cpu, yet do not wish to teardown the dma mapping, you must
888 * call this function before doing so. At the next point you give the dma
889 * address back to the card, you must first perform a
890 * swiotlb_dma_sync_for_device, and then the device again owns the buffer
891 */
892static void
893swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
894 size_t size, enum dma_data_direction dir,
895 enum dma_sync_target target)
896{
897 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
898
899 BUG_ON(dir == DMA_NONE);
900
901 if (is_swiotlb_buffer(paddr)) {
902 swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);
903 return;
904 }
905
906 if (dir != DMA_FROM_DEVICE)
907 return;
908
909 dma_mark_clean(phys_to_virt(paddr), size);
910}
911
912void
913swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
914 size_t size, enum dma_data_direction dir)
915{
916 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
917}
918
919void
920swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
921 size_t size, enum dma_data_direction dir)
922{
923 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
924}
925
926/*
927 * Map a set of buffers described by scatterlist in streaming mode for DMA.
928 * This is the scatter-gather version of the above swiotlb_map_page
929 * interface. Here the scatter gather list elements are each tagged with the
930 * appropriate dma address and length. They are obtained via
931 * sg_dma_{address,length}(SG).
932 *
933 * NOTE: An implementation may be able to use a smaller number of
934 * DMA address/length pairs than there are SG table elements.
935 * (for example via virtual mapping capabilities)
936 * The routine returns the number of addr/length pairs actually
937 * used, at most nents.
938 *
939 * Device ownership issues as mentioned above for swiotlb_map_page are the
940 * same here.
941 */
942int
943swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
944 enum dma_data_direction dir, unsigned long attrs)
945{
946 struct scatterlist *sg;
947 int i;
948
949 BUG_ON(dir == DMA_NONE);
950
951 for_each_sg(sgl, sg, nelems, i) {
952 phys_addr_t paddr = sg_phys(sg);
953 dma_addr_t dev_addr = phys_to_dma(hwdev, paddr);
954
955 if (swiotlb_force == SWIOTLB_FORCE ||
956 !dma_capable(hwdev, dev_addr, sg->length)) {
957 phys_addr_t map = map_single(hwdev, sg_phys(sg),
958 sg->length, dir, attrs);
959 if (map == SWIOTLB_MAP_ERROR) {
960 /* Don't panic here, we expect map_sg users
961 to do proper error handling. */
962 swiotlb_full(hwdev, sg->length, dir, 0);
963 attrs |= DMA_ATTR_SKIP_CPU_SYNC;
964 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
965 attrs);
966 sg_dma_len(sgl) = 0;
967 return 0;
968 }
969 sg->dma_address = __phys_to_dma(hwdev, map);
970 } else
971 sg->dma_address = dev_addr;
972 sg_dma_len(sg) = sg->length;
973 }
974 return nelems;
975}
976
977/*
978 * Unmap a set of streaming mode DMA translations. Again, cpu read rules
979 * concerning calls here are the same as for swiotlb_unmap_page() above.
980 */
981void
982swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
983 int nelems, enum dma_data_direction dir,
984 unsigned long attrs)
985{
986 struct scatterlist *sg;
987 int i;
988
989 BUG_ON(dir == DMA_NONE);
990
991 for_each_sg(sgl, sg, nelems, i)
992 unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir,
993 attrs);
994}
995
996/*
997 * Make physical memory consistent for a set of streaming mode DMA translations
998 * after a transfer.
999 *
1000 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
1001 * and usage.
1002 */
1003static void
1004swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
1005 int nelems, enum dma_data_direction dir,
1006 enum dma_sync_target target)
1007{
1008 struct scatterlist *sg;
1009 int i;
1010
1011 for_each_sg(sgl, sg, nelems, i)
1012 swiotlb_sync_single(hwdev, sg->dma_address,
1013 sg_dma_len(sg), dir, target);
1014}
1015
1016void
1017swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
1018 int nelems, enum dma_data_direction dir)
1019{
1020 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
1021}
1022
1023void
1024swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
1025 int nelems, enum dma_data_direction dir)
1026{
1027 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
1028}
1029
1030int
1031swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
1032{
1033 return (dma_addr == __phys_to_dma(hwdev, io_tlb_overflow_buffer));
1034}
1035
1036/*
1037 * Return whether the given device DMA address mask can be supported
1038 * properly. For example, if your device can only drive the low 24-bits
1039 * during bus mastering, then you would pass 0x00ffffff as the mask to
1040 * this function.
1041 */
1042int
1043swiotlb_dma_supported(struct device *hwdev, u64 mask)
1044{
1045 return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
1046}
1047
1048#ifdef CONFIG_DMA_DIRECT_OPS
1049void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
1050 gfp_t gfp, unsigned long attrs)
1051{
1052 void *vaddr;
1053
1054 /* temporary workaround: */
1055 if (gfp & __GFP_NOWARN)
1056 attrs |= DMA_ATTR_NO_WARN;
1057
1058 /*
1059 * Don't print a warning when the first allocation attempt fails.
1060 * swiotlb_alloc_coherent() will print a warning when the DMA memory
1061 * allocation ultimately failed.
1062 */
1063 gfp |= __GFP_NOWARN;
1064
1065 vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
1066 if (!vaddr)
1067 vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs);
1068 return vaddr;
1069}
1070
1071void swiotlb_free(struct device *dev, size_t size, void *vaddr,
1072 dma_addr_t dma_addr, unsigned long attrs)
1073{
1074 if (!swiotlb_free_buffer(dev, size, dma_addr))
1075 dma_direct_free(dev, size, vaddr, dma_addr, attrs);
1076}
1077
1078const struct dma_map_ops swiotlb_dma_ops = {
1079 .mapping_error = swiotlb_dma_mapping_error,
1080 .alloc = swiotlb_alloc,
1081 .free = swiotlb_free,
1082 .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
1083 .sync_single_for_device = swiotlb_sync_single_for_device,
1084 .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
1085 .sync_sg_for_device = swiotlb_sync_sg_for_device,
1086 .map_sg = swiotlb_map_sg_attrs,
1087 .unmap_sg = swiotlb_unmap_sg_attrs,
1088 .map_page = swiotlb_map_page,
1089 .unmap_page = swiotlb_unmap_page,
1090 .dma_supported = dma_direct_supported,
1091};
1092#endif /* CONFIG_DMA_DIRECT_OPS */
diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c
index de16f7869fb1..6cd7d0740005 100644
--- a/lib/test_bitmap.c
+++ b/lib/test_bitmap.c
@@ -331,23 +331,32 @@ static void noinline __init test_mem_optimisations(void)
331 unsigned int start, nbits; 331 unsigned int start, nbits;
332 332
333 for (start = 0; start < 1024; start += 8) { 333 for (start = 0; start < 1024; start += 8) {
334 memset(bmap1, 0x5a, sizeof(bmap1));
335 memset(bmap2, 0x5a, sizeof(bmap2));
336 for (nbits = 0; nbits < 1024 - start; nbits += 8) { 334 for (nbits = 0; nbits < 1024 - start; nbits += 8) {
335 memset(bmap1, 0x5a, sizeof(bmap1));
336 memset(bmap2, 0x5a, sizeof(bmap2));
337
337 bitmap_set(bmap1, start, nbits); 338 bitmap_set(bmap1, start, nbits);
338 __bitmap_set(bmap2, start, nbits); 339 __bitmap_set(bmap2, start, nbits);
339 if (!bitmap_equal(bmap1, bmap2, 1024)) 340 if (!bitmap_equal(bmap1, bmap2, 1024)) {
340 printk("set not equal %d %d\n", start, nbits); 341 printk("set not equal %d %d\n", start, nbits);
341 if (!__bitmap_equal(bmap1, bmap2, 1024)) 342 failed_tests++;
343 }
344 if (!__bitmap_equal(bmap1, bmap2, 1024)) {
342 printk("set not __equal %d %d\n", start, nbits); 345 printk("set not __equal %d %d\n", start, nbits);
346 failed_tests++;
347 }
343 348
344 bitmap_clear(bmap1, start, nbits); 349 bitmap_clear(bmap1, start, nbits);
345 __bitmap_clear(bmap2, start, nbits); 350 __bitmap_clear(bmap2, start, nbits);
346 if (!bitmap_equal(bmap1, bmap2, 1024)) 351 if (!bitmap_equal(bmap1, bmap2, 1024)) {
347 printk("clear not equal %d %d\n", start, nbits); 352 printk("clear not equal %d %d\n", start, nbits);
348 if (!__bitmap_equal(bmap1, bmap2, 1024)) 353 failed_tests++;
354 }
355 if (!__bitmap_equal(bmap1, bmap2, 1024)) {
349 printk("clear not __equal %d %d\n", start, 356 printk("clear not __equal %d %d\n", start,
350 nbits); 357 nbits);
358 failed_tests++;
359 }
351 } 360 }
352 } 361 }
353} 362}
diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index 8e157806df7a..08d3d59dca17 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -356,29 +356,22 @@ static int bpf_fill_maxinsns11(struct bpf_test *self)
356 return __bpf_fill_ja(self, BPF_MAXINSNS, 68); 356 return __bpf_fill_ja(self, BPF_MAXINSNS, 68);
357} 357}
358 358
359static int bpf_fill_ja(struct bpf_test *self) 359static int bpf_fill_maxinsns12(struct bpf_test *self)
360{
361 /* Hits exactly 11 passes on x86_64 JIT. */
362 return __bpf_fill_ja(self, 12, 9);
363}
364
365static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self)
366{ 360{
367 unsigned int len = BPF_MAXINSNS; 361 unsigned int len = BPF_MAXINSNS;
368 struct sock_filter *insn; 362 struct sock_filter *insn;
369 int i; 363 int i = 0;
370 364
371 insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); 365 insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
372 if (!insn) 366 if (!insn)
373 return -ENOMEM; 367 return -ENOMEM;
374 368
375 for (i = 0; i < len - 1; i += 2) { 369 insn[0] = __BPF_JUMP(BPF_JMP | BPF_JA, len - 2, 0, 0);
376 insn[i] = __BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 0);
377 insn[i + 1] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS,
378 SKF_AD_OFF + SKF_AD_CPU);
379 }
380 370
381 insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xbee); 371 for (i = 1; i < len - 1; i++)
372 insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0);
373
374 insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xabababab);
382 375
383 self->u.ptr.insns = insn; 376 self->u.ptr.insns = insn;
384 self->u.ptr.len = len; 377 self->u.ptr.len = len;
@@ -386,50 +379,22 @@ static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self)
386 return 0; 379 return 0;
387} 380}
388 381
389#define PUSH_CNT 68 382static int bpf_fill_maxinsns13(struct bpf_test *self)
390/* test: {skb->data[0], vlan_push} x 68 + {skb->data[0], vlan_pop} x 68 */
391static int bpf_fill_ld_abs_vlan_push_pop(struct bpf_test *self)
392{ 383{
393 unsigned int len = BPF_MAXINSNS; 384 unsigned int len = BPF_MAXINSNS;
394 struct bpf_insn *insn; 385 struct sock_filter *insn;
395 int i = 0, j, k = 0; 386 int i = 0;
396 387
397 insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); 388 insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
398 if (!insn) 389 if (!insn)
399 return -ENOMEM; 390 return -ENOMEM;
400 391
401 insn[i++] = BPF_MOV64_REG(R6, R1); 392 for (i = 0; i < len - 3; i++)
402loop: 393 insn[i] = __BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0);
403 for (j = 0; j < PUSH_CNT; j++) {
404 insn[i++] = BPF_LD_ABS(BPF_B, 0);
405 insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0x34, len - i - 2);
406 i++;
407 insn[i++] = BPF_MOV64_REG(R1, R6);
408 insn[i++] = BPF_MOV64_IMM(R2, 1);
409 insn[i++] = BPF_MOV64_IMM(R3, 2);
410 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
411 bpf_skb_vlan_push_proto.func - __bpf_call_base);
412 insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0, len - i - 2);
413 i++;
414 }
415 394
416 for (j = 0; j < PUSH_CNT; j++) { 395 insn[len - 3] = __BPF_STMT(BPF_LD | BPF_IMM, 0xabababab);
417 insn[i++] = BPF_LD_ABS(BPF_B, 0); 396 insn[len - 2] = __BPF_STMT(BPF_ALU | BPF_XOR | BPF_X, 0);
418 insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0x34, len - i - 2); 397 insn[len - 1] = __BPF_STMT(BPF_RET | BPF_A, 0);
419 i++;
420 insn[i++] = BPF_MOV64_REG(R1, R6);
421 insn[i++] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
422 bpf_skb_vlan_pop_proto.func - __bpf_call_base);
423 insn[i] = BPF_JMP_IMM(BPF_JNE, R0, 0, len - i - 2);
424 i++;
425 }
426 if (++k < 5)
427 goto loop;
428
429 for (; i < len - 1; i++)
430 insn[i] = BPF_ALU32_IMM(BPF_MOV, R0, 0xbef);
431
432 insn[len - 1] = BPF_EXIT_INSN();
433 398
434 self->u.ptr.insns = insn; 399 self->u.ptr.insns = insn;
435 self->u.ptr.len = len; 400 self->u.ptr.len = len;
@@ -437,58 +402,29 @@ loop:
437 return 0; 402 return 0;
438} 403}
439 404
440static int bpf_fill_ld_abs_vlan_push_pop2(struct bpf_test *self) 405static int bpf_fill_ja(struct bpf_test *self)
441{ 406{
442 struct bpf_insn *insn; 407 /* Hits exactly 11 passes on x86_64 JIT. */
443 408 return __bpf_fill_ja(self, 12, 9);
444 insn = kmalloc_array(16, sizeof(*insn), GFP_KERNEL);
445 if (!insn)
446 return -ENOMEM;
447
448 /* Due to func address being non-const, we need to
449 * assemble this here.
450 */
451 insn[0] = BPF_MOV64_REG(R6, R1);
452 insn[1] = BPF_LD_ABS(BPF_B, 0);
453 insn[2] = BPF_LD_ABS(BPF_H, 0);
454 insn[3] = BPF_LD_ABS(BPF_W, 0);
455 insn[4] = BPF_MOV64_REG(R7, R6);
456 insn[5] = BPF_MOV64_IMM(R6, 0);
457 insn[6] = BPF_MOV64_REG(R1, R7);
458 insn[7] = BPF_MOV64_IMM(R2, 1);
459 insn[8] = BPF_MOV64_IMM(R3, 2);
460 insn[9] = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
461 bpf_skb_vlan_push_proto.func - __bpf_call_base);
462 insn[10] = BPF_MOV64_REG(R6, R7);
463 insn[11] = BPF_LD_ABS(BPF_B, 0);
464 insn[12] = BPF_LD_ABS(BPF_H, 0);
465 insn[13] = BPF_LD_ABS(BPF_W, 0);
466 insn[14] = BPF_MOV64_IMM(R0, 42);
467 insn[15] = BPF_EXIT_INSN();
468
469 self->u.ptr.insns = insn;
470 self->u.ptr.len = 16;
471
472 return 0;
473} 409}
474 410
475static int bpf_fill_jump_around_ld_abs(struct bpf_test *self) 411static int bpf_fill_ld_abs_get_processor_id(struct bpf_test *self)
476{ 412{
477 unsigned int len = BPF_MAXINSNS; 413 unsigned int len = BPF_MAXINSNS;
478 struct bpf_insn *insn; 414 struct sock_filter *insn;
479 int i = 0; 415 int i;
480 416
481 insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); 417 insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
482 if (!insn) 418 if (!insn)
483 return -ENOMEM; 419 return -ENOMEM;
484 420
485 insn[i++] = BPF_MOV64_REG(R6, R1); 421 for (i = 0; i < len - 1; i += 2) {
486 insn[i++] = BPF_LD_ABS(BPF_B, 0); 422 insn[i] = __BPF_STMT(BPF_LD | BPF_B | BPF_ABS, 0);
487 insn[i] = BPF_JMP_IMM(BPF_JEQ, R0, 10, len - i - 2); 423 insn[i + 1] = __BPF_STMT(BPF_LD | BPF_W | BPF_ABS,
488 i++; 424 SKF_AD_OFF + SKF_AD_CPU);
489 while (i < len - 1) 425 }
490 insn[i++] = BPF_LD_ABS(BPF_B, 1); 426
491 insn[i] = BPF_EXIT_INSN(); 427 insn[len - 1] = __BPF_STMT(BPF_RET | BPF_K, 0xbee);
492 428
493 self->u.ptr.insns = insn; 429 self->u.ptr.insns = insn;
494 self->u.ptr.len = len; 430 self->u.ptr.len = len;
@@ -1988,40 +1924,6 @@ static struct bpf_test tests[] = {
1988 { { 0, -1 } } 1924 { { 0, -1 } }
1989 }, 1925 },
1990 { 1926 {
1991 "INT: DIV + ABS",
1992 .u.insns_int = {
1993 BPF_ALU64_REG(BPF_MOV, R6, R1),
1994 BPF_LD_ABS(BPF_B, 3),
1995 BPF_ALU64_IMM(BPF_MOV, R2, 2),
1996 BPF_ALU32_REG(BPF_DIV, R0, R2),
1997 BPF_ALU64_REG(BPF_MOV, R8, R0),
1998 BPF_LD_ABS(BPF_B, 4),
1999 BPF_ALU64_REG(BPF_ADD, R8, R0),
2000 BPF_LD_IND(BPF_B, R8, -70),
2001 BPF_EXIT_INSN(),
2002 },
2003 INTERNAL,
2004 { 10, 20, 30, 40, 50 },
2005 { { 4, 0 }, { 5, 10 } }
2006 },
2007 {
2008 /* This one doesn't go through verifier, but is just raw insn
2009 * as opposed to cBPF tests from here. Thus div by 0 tests are
2010 * done in test_verifier in BPF kselftests.
2011 */
2012 "INT: DIV by -1",
2013 .u.insns_int = {
2014 BPF_ALU64_REG(BPF_MOV, R6, R1),
2015 BPF_ALU64_IMM(BPF_MOV, R7, -1),
2016 BPF_LD_ABS(BPF_B, 3),
2017 BPF_ALU32_REG(BPF_DIV, R0, R7),
2018 BPF_EXIT_INSN(),
2019 },
2020 INTERNAL,
2021 { 10, 20, 30, 40, 50 },
2022 { { 3, 0 }, { 4, 0 } }
2023 },
2024 {
2025 "check: missing ret", 1927 "check: missing ret",
2026 .u.insns = { 1928 .u.insns = {
2027 BPF_STMT(BPF_LD | BPF_IMM, 1), 1929 BPF_STMT(BPF_LD | BPF_IMM, 1),
@@ -2383,50 +2285,6 @@ static struct bpf_test tests[] = {
2383 { }, 2285 { },
2384 { { 0, 1 } } 2286 { { 0, 1 } }
2385 }, 2287 },
2386 {
2387 "nmap reduced",
2388 .u.insns_int = {
2389 BPF_MOV64_REG(R6, R1),
2390 BPF_LD_ABS(BPF_H, 12),
2391 BPF_JMP_IMM(BPF_JNE, R0, 0x806, 28),
2392 BPF_LD_ABS(BPF_H, 12),
2393 BPF_JMP_IMM(BPF_JNE, R0, 0x806, 26),
2394 BPF_MOV32_IMM(R0, 18),
2395 BPF_STX_MEM(BPF_W, R10, R0, -64),
2396 BPF_LDX_MEM(BPF_W, R7, R10, -64),
2397 BPF_LD_IND(BPF_W, R7, 14),
2398 BPF_STX_MEM(BPF_W, R10, R0, -60),
2399 BPF_MOV32_IMM(R0, 280971478),
2400 BPF_STX_MEM(BPF_W, R10, R0, -56),
2401 BPF_LDX_MEM(BPF_W, R7, R10, -56),
2402 BPF_LDX_MEM(BPF_W, R0, R10, -60),
2403 BPF_ALU32_REG(BPF_SUB, R0, R7),
2404 BPF_JMP_IMM(BPF_JNE, R0, 0, 15),
2405 BPF_LD_ABS(BPF_H, 12),
2406 BPF_JMP_IMM(BPF_JNE, R0, 0x806, 13),
2407 BPF_MOV32_IMM(R0, 22),
2408 BPF_STX_MEM(BPF_W, R10, R0, -56),
2409 BPF_LDX_MEM(BPF_W, R7, R10, -56),
2410 BPF_LD_IND(BPF_H, R7, 14),
2411 BPF_STX_MEM(BPF_W, R10, R0, -52),
2412 BPF_MOV32_IMM(R0, 17366),
2413 BPF_STX_MEM(BPF_W, R10, R0, -48),
2414 BPF_LDX_MEM(BPF_W, R7, R10, -48),
2415 BPF_LDX_MEM(BPF_W, R0, R10, -52),
2416 BPF_ALU32_REG(BPF_SUB, R0, R7),
2417 BPF_JMP_IMM(BPF_JNE, R0, 0, 2),
2418 BPF_MOV32_IMM(R0, 256),
2419 BPF_EXIT_INSN(),
2420 BPF_MOV32_IMM(R0, 0),
2421 BPF_EXIT_INSN(),
2422 },
2423 INTERNAL,
2424 { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x06, 0, 0,
2425 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
2426 0x10, 0xbf, 0x48, 0xd6, 0x43, 0xd6},
2427 { { 38, 256 } },
2428 .stack_depth = 64,
2429 },
2430 /* BPF_ALU | BPF_MOV | BPF_X */ 2288 /* BPF_ALU | BPF_MOV | BPF_X */
2431 { 2289 {
2432 "ALU_MOV_X: dst = 2", 2290 "ALU_MOV_X: dst = 2",
@@ -5424,21 +5282,31 @@ static struct bpf_test tests[] = {
5424 { /* Mainly checking JIT here. */ 5282 { /* Mainly checking JIT here. */
5425 "BPF_MAXINSNS: Ctx heavy transformations", 5283 "BPF_MAXINSNS: Ctx heavy transformations",
5426 { }, 5284 { },
5285#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390)
5286 CLASSIC | FLAG_EXPECTED_FAIL,
5287#else
5427 CLASSIC, 5288 CLASSIC,
5289#endif
5428 { }, 5290 { },
5429 { 5291 {
5430 { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }, 5292 { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) },
5431 { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) } 5293 { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }
5432 }, 5294 },
5433 .fill_helper = bpf_fill_maxinsns6, 5295 .fill_helper = bpf_fill_maxinsns6,
5296 .expected_errcode = -ENOTSUPP,
5434 }, 5297 },
5435 { /* Mainly checking JIT here. */ 5298 { /* Mainly checking JIT here. */
5436 "BPF_MAXINSNS: Call heavy transformations", 5299 "BPF_MAXINSNS: Call heavy transformations",
5437 { }, 5300 { },
5301#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390)
5302 CLASSIC | FLAG_NO_DATA | FLAG_EXPECTED_FAIL,
5303#else
5438 CLASSIC | FLAG_NO_DATA, 5304 CLASSIC | FLAG_NO_DATA,
5305#endif
5439 { }, 5306 { },
5440 { { 1, 0 }, { 10, 0 } }, 5307 { { 1, 0 }, { 10, 0 } },
5441 .fill_helper = bpf_fill_maxinsns7, 5308 .fill_helper = bpf_fill_maxinsns7,
5309 .expected_errcode = -ENOTSUPP,
5442 }, 5310 },
5443 { /* Mainly checking JIT here. */ 5311 { /* Mainly checking JIT here. */
5444 "BPF_MAXINSNS: Jump heavy test", 5312 "BPF_MAXINSNS: Jump heavy test",
@@ -5478,28 +5346,39 @@ static struct bpf_test tests[] = {
5478 .expected_errcode = -ENOTSUPP, 5346 .expected_errcode = -ENOTSUPP,
5479 }, 5347 },
5480 { 5348 {
5481 "BPF_MAXINSNS: ld_abs+get_processor_id", 5349 "BPF_MAXINSNS: jump over MSH",
5482 { },
5483 CLASSIC,
5484 { }, 5350 { },
5485 { { 1, 0xbee } }, 5351 CLASSIC | FLAG_EXPECTED_FAIL,
5486 .fill_helper = bpf_fill_ld_abs_get_processor_id, 5352 { 0xfa, 0xfb, 0xfc, 0xfd, },
5353 { { 4, 0xabababab } },
5354 .fill_helper = bpf_fill_maxinsns12,
5355 .expected_errcode = -EINVAL,
5487 }, 5356 },
5488 { 5357 {
5489 "BPF_MAXINSNS: ld_abs+vlan_push/pop", 5358 "BPF_MAXINSNS: exec all MSH",
5490 { }, 5359 { },
5491 INTERNAL, 5360#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390)
5492 { 0x34 }, 5361 CLASSIC | FLAG_EXPECTED_FAIL,
5493 { { ETH_HLEN, 0xbef } }, 5362#else
5494 .fill_helper = bpf_fill_ld_abs_vlan_push_pop, 5363 CLASSIC,
5364#endif
5365 { 0xfa, 0xfb, 0xfc, 0xfd, },
5366 { { 4, 0xababab83 } },
5367 .fill_helper = bpf_fill_maxinsns13,
5368 .expected_errcode = -ENOTSUPP,
5495 }, 5369 },
5496 { 5370 {
5497 "BPF_MAXINSNS: jump around ld_abs", 5371 "BPF_MAXINSNS: ld_abs+get_processor_id",
5498 { }, 5372 { },
5499 INTERNAL, 5373#if defined(CONFIG_BPF_JIT_ALWAYS_ON) && defined(CONFIG_S390)
5500 { 10, 11 }, 5374 CLASSIC | FLAG_EXPECTED_FAIL,
5501 { { 2, 10 } }, 5375#else
5502 .fill_helper = bpf_fill_jump_around_ld_abs, 5376 CLASSIC,
5377#endif
5378 { },
5379 { { 1, 0xbee } },
5380 .fill_helper = bpf_fill_ld_abs_get_processor_id,
5381 .expected_errcode = -ENOTSUPP,
5503 }, 5382 },
5504 /* 5383 /*
5505 * LD_IND / LD_ABS on fragmented SKBs 5384 * LD_IND / LD_ABS on fragmented SKBs
@@ -5683,6 +5562,53 @@ static struct bpf_test tests[] = {
5683 { {0x40, 0x05 } }, 5562 { {0x40, 0x05 } },
5684 }, 5563 },
5685 { 5564 {
5565 "LD_IND byte positive offset, all ff",
5566 .u.insns = {
5567 BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
5568 BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1),
5569 BPF_STMT(BPF_RET | BPF_A, 0x0),
5570 },
5571 CLASSIC,
5572 { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
5573 { {0x40, 0xff } },
5574 },
5575 {
5576 "LD_IND byte positive offset, out of bounds",
5577 .u.insns = {
5578 BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
5579 BPF_STMT(BPF_LD | BPF_IND | BPF_B, 0x1),
5580 BPF_STMT(BPF_RET | BPF_A, 0x0),
5581 },
5582 CLASSIC,
5583 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5584 { {0x3f, 0 }, },
5585 },
5586 {
5587 "LD_IND byte negative offset, out of bounds",
5588 .u.insns = {
5589 BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
5590 BPF_STMT(BPF_LD | BPF_IND | BPF_B, -0x3f),
5591 BPF_STMT(BPF_RET | BPF_A, 0x0),
5592 },
5593 CLASSIC,
5594 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5595 { {0x3f, 0 } },
5596 },
5597 {
5598 "LD_IND byte negative offset, multiple calls",
5599 .u.insns = {
5600 BPF_STMT(BPF_LDX | BPF_IMM, 0x3b),
5601 BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 1),
5602 BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 2),
5603 BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 3),
5604 BPF_STMT(BPF_LD | BPF_IND | BPF_B, SKF_LL_OFF + 4),
5605 BPF_STMT(BPF_RET | BPF_A, 0x0),
5606 },
5607 CLASSIC,
5608 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5609 { {0x40, 0x82 }, },
5610 },
5611 {
5686 "LD_IND halfword positive offset", 5612 "LD_IND halfword positive offset",
5687 .u.insns = { 5613 .u.insns = {
5688 BPF_STMT(BPF_LDX | BPF_IMM, 0x20), 5614 BPF_STMT(BPF_LDX | BPF_IMM, 0x20),
@@ -5731,6 +5657,39 @@ static struct bpf_test tests[] = {
5731 { {0x40, 0x66cc } }, 5657 { {0x40, 0x66cc } },
5732 }, 5658 },
5733 { 5659 {
5660 "LD_IND halfword positive offset, all ff",
5661 .u.insns = {
5662 BPF_STMT(BPF_LDX | BPF_IMM, 0x3d),
5663 BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1),
5664 BPF_STMT(BPF_RET | BPF_A, 0x0),
5665 },
5666 CLASSIC,
5667 { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
5668 { {0x40, 0xffff } },
5669 },
5670 {
5671 "LD_IND halfword positive offset, out of bounds",
5672 .u.insns = {
5673 BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
5674 BPF_STMT(BPF_LD | BPF_IND | BPF_H, 0x1),
5675 BPF_STMT(BPF_RET | BPF_A, 0x0),
5676 },
5677 CLASSIC,
5678 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5679 { {0x3f, 0 }, },
5680 },
5681 {
5682 "LD_IND halfword negative offset, out of bounds",
5683 .u.insns = {
5684 BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
5685 BPF_STMT(BPF_LD | BPF_IND | BPF_H, -0x3f),
5686 BPF_STMT(BPF_RET | BPF_A, 0x0),
5687 },
5688 CLASSIC,
5689 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5690 { {0x3f, 0 } },
5691 },
5692 {
5734 "LD_IND word positive offset", 5693 "LD_IND word positive offset",
5735 .u.insns = { 5694 .u.insns = {
5736 BPF_STMT(BPF_LDX | BPF_IMM, 0x20), 5695 BPF_STMT(BPF_LDX | BPF_IMM, 0x20),
@@ -5821,6 +5780,39 @@ static struct bpf_test tests[] = {
5821 { {0x40, 0x66cc77dd } }, 5780 { {0x40, 0x66cc77dd } },
5822 }, 5781 },
5823 { 5782 {
5783 "LD_IND word positive offset, all ff",
5784 .u.insns = {
5785 BPF_STMT(BPF_LDX | BPF_IMM, 0x3b),
5786 BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1),
5787 BPF_STMT(BPF_RET | BPF_A, 0x0),
5788 },
5789 CLASSIC,
5790 { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
5791 { {0x40, 0xffffffff } },
5792 },
5793 {
5794 "LD_IND word positive offset, out of bounds",
5795 .u.insns = {
5796 BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
5797 BPF_STMT(BPF_LD | BPF_IND | BPF_W, 0x1),
5798 BPF_STMT(BPF_RET | BPF_A, 0x0),
5799 },
5800 CLASSIC,
5801 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5802 { {0x3f, 0 }, },
5803 },
5804 {
5805 "LD_IND word negative offset, out of bounds",
5806 .u.insns = {
5807 BPF_STMT(BPF_LDX | BPF_IMM, 0x3e),
5808 BPF_STMT(BPF_LD | BPF_IND | BPF_W, -0x3f),
5809 BPF_STMT(BPF_RET | BPF_A, 0x0),
5810 },
5811 CLASSIC,
5812 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5813 { {0x3f, 0 } },
5814 },
5815 {
5824 "LD_ABS byte", 5816 "LD_ABS byte",
5825 .u.insns = { 5817 .u.insns = {
5826 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x20), 5818 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x20),
@@ -5838,6 +5830,68 @@ static struct bpf_test tests[] = {
5838 { {0x40, 0xcc } }, 5830 { {0x40, 0xcc } },
5839 }, 5831 },
5840 { 5832 {
5833 "LD_ABS byte positive offset, all ff",
5834 .u.insns = {
5835 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f),
5836 BPF_STMT(BPF_RET | BPF_A, 0x0),
5837 },
5838 CLASSIC,
5839 { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
5840 { {0x40, 0xff } },
5841 },
5842 {
5843 "LD_ABS byte positive offset, out of bounds",
5844 .u.insns = {
5845 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, 0x3f),
5846 BPF_STMT(BPF_RET | BPF_A, 0x0),
5847 },
5848 CLASSIC,
5849 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5850 { {0x3f, 0 }, },
5851 },
5852 {
5853 "LD_ABS byte negative offset, out of bounds load",
5854 .u.insns = {
5855 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, -1),
5856 BPF_STMT(BPF_RET | BPF_A, 0x0),
5857 },
5858 CLASSIC | FLAG_EXPECTED_FAIL,
5859 .expected_errcode = -EINVAL,
5860 },
5861 {
5862 "LD_ABS byte negative offset, in bounds",
5863 .u.insns = {
5864 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f),
5865 BPF_STMT(BPF_RET | BPF_A, 0x0),
5866 },
5867 CLASSIC,
5868 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5869 { {0x40, 0x82 }, },
5870 },
5871 {
5872 "LD_ABS byte negative offset, out of bounds",
5873 .u.insns = {
5874 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f),
5875 BPF_STMT(BPF_RET | BPF_A, 0x0),
5876 },
5877 CLASSIC,
5878 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5879 { {0x3f, 0 }, },
5880 },
5881 {
5882 "LD_ABS byte negative offset, multiple calls",
5883 .u.insns = {
5884 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3c),
5885 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3d),
5886 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3e),
5887 BPF_STMT(BPF_LD | BPF_ABS | BPF_B, SKF_LL_OFF + 0x3f),
5888 BPF_STMT(BPF_RET | BPF_A, 0x0),
5889 },
5890 CLASSIC,
5891 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5892 { {0x40, 0x82 }, },
5893 },
5894 {
5841 "LD_ABS halfword", 5895 "LD_ABS halfword",
5842 .u.insns = { 5896 .u.insns = {
5843 BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x22), 5897 BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x22),
@@ -5872,6 +5926,55 @@ static struct bpf_test tests[] = {
5872 { {0x40, 0x99ff } }, 5926 { {0x40, 0x99ff } },
5873 }, 5927 },
5874 { 5928 {
5929 "LD_ABS halfword positive offset, all ff",
5930 .u.insns = {
5931 BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3e),
5932 BPF_STMT(BPF_RET | BPF_A, 0x0),
5933 },
5934 CLASSIC,
5935 { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
5936 { {0x40, 0xffff } },
5937 },
5938 {
5939 "LD_ABS halfword positive offset, out of bounds",
5940 .u.insns = {
5941 BPF_STMT(BPF_LD | BPF_ABS | BPF_H, 0x3f),
5942 BPF_STMT(BPF_RET | BPF_A, 0x0),
5943 },
5944 CLASSIC,
5945 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5946 { {0x3f, 0 }, },
5947 },
5948 {
5949 "LD_ABS halfword negative offset, out of bounds load",
5950 .u.insns = {
5951 BPF_STMT(BPF_LD | BPF_ABS | BPF_H, -1),
5952 BPF_STMT(BPF_RET | BPF_A, 0x0),
5953 },
5954 CLASSIC | FLAG_EXPECTED_FAIL,
5955 .expected_errcode = -EINVAL,
5956 },
5957 {
5958 "LD_ABS halfword negative offset, in bounds",
5959 .u.insns = {
5960 BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e),
5961 BPF_STMT(BPF_RET | BPF_A, 0x0),
5962 },
5963 CLASSIC,
5964 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5965 { {0x40, 0x1982 }, },
5966 },
5967 {
5968 "LD_ABS halfword negative offset, out of bounds",
5969 .u.insns = {
5970 BPF_STMT(BPF_LD | BPF_ABS | BPF_H, SKF_LL_OFF + 0x3e),
5971 BPF_STMT(BPF_RET | BPF_A, 0x0),
5972 },
5973 CLASSIC,
5974 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
5975 { {0x3f, 0 }, },
5976 },
5977 {
5875 "LD_ABS word", 5978 "LD_ABS word",
5876 .u.insns = { 5979 .u.insns = {
5877 BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x1c), 5980 BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x1c),
@@ -5939,6 +6042,140 @@ static struct bpf_test tests[] = {
5939 }, 6042 },
5940 { {0x40, 0x88ee99ff } }, 6043 { {0x40, 0x88ee99ff } },
5941 }, 6044 },
6045 {
6046 "LD_ABS word positive offset, all ff",
6047 .u.insns = {
6048 BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3c),
6049 BPF_STMT(BPF_RET | BPF_A, 0x0),
6050 },
6051 CLASSIC,
6052 { [0x3c] = 0xff, [0x3d] = 0xff, [0x3e] = 0xff, [0x3f] = 0xff },
6053 { {0x40, 0xffffffff } },
6054 },
6055 {
6056 "LD_ABS word positive offset, out of bounds",
6057 .u.insns = {
6058 BPF_STMT(BPF_LD | BPF_ABS | BPF_W, 0x3f),
6059 BPF_STMT(BPF_RET | BPF_A, 0x0),
6060 },
6061 CLASSIC,
6062 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6063 { {0x3f, 0 }, },
6064 },
6065 {
6066 "LD_ABS word negative offset, out of bounds load",
6067 .u.insns = {
6068 BPF_STMT(BPF_LD | BPF_ABS | BPF_W, -1),
6069 BPF_STMT(BPF_RET | BPF_A, 0x0),
6070 },
6071 CLASSIC | FLAG_EXPECTED_FAIL,
6072 .expected_errcode = -EINVAL,
6073 },
6074 {
6075 "LD_ABS word negative offset, in bounds",
6076 .u.insns = {
6077 BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c),
6078 BPF_STMT(BPF_RET | BPF_A, 0x0),
6079 },
6080 CLASSIC,
6081 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6082 { {0x40, 0x25051982 }, },
6083 },
6084 {
6085 "LD_ABS word negative offset, out of bounds",
6086 .u.insns = {
6087 BPF_STMT(BPF_LD | BPF_ABS | BPF_W, SKF_LL_OFF + 0x3c),
6088 BPF_STMT(BPF_RET | BPF_A, 0x0),
6089 },
6090 CLASSIC,
6091 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6092 { {0x3f, 0 }, },
6093 },
6094 {
6095 "LDX_MSH standalone, preserved A",
6096 .u.insns = {
6097 BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
6098 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c),
6099 BPF_STMT(BPF_RET | BPF_A, 0x0),
6100 },
6101 CLASSIC,
6102 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6103 { {0x40, 0xffeebbaa }, },
6104 },
6105 {
6106 "LDX_MSH standalone, preserved A 2",
6107 .u.insns = {
6108 BPF_STMT(BPF_LD | BPF_IMM, 0x175e9d63),
6109 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c),
6110 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3d),
6111 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e),
6112 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3f),
6113 BPF_STMT(BPF_RET | BPF_A, 0x0),
6114 },
6115 CLASSIC,
6116 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6117 { {0x40, 0x175e9d63 }, },
6118 },
6119 {
6120 "LDX_MSH standalone, test result 1",
6121 .u.insns = {
6122 BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
6123 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3c),
6124 BPF_STMT(BPF_MISC | BPF_TXA, 0),
6125 BPF_STMT(BPF_RET | BPF_A, 0x0),
6126 },
6127 CLASSIC,
6128 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6129 { {0x40, 0x14 }, },
6130 },
6131 {
6132 "LDX_MSH standalone, test result 2",
6133 .u.insns = {
6134 BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
6135 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x3e),
6136 BPF_STMT(BPF_MISC | BPF_TXA, 0),
6137 BPF_STMT(BPF_RET | BPF_A, 0x0),
6138 },
6139 CLASSIC,
6140 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6141 { {0x40, 0x24 }, },
6142 },
6143 {
6144 "LDX_MSH standalone, negative offset",
6145 .u.insns = {
6146 BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
6147 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, -1),
6148 BPF_STMT(BPF_MISC | BPF_TXA, 0),
6149 BPF_STMT(BPF_RET | BPF_A, 0x0),
6150 },
6151 CLASSIC,
6152 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6153 { {0x40, 0 }, },
6154 },
6155 {
6156 "LDX_MSH standalone, negative offset 2",
6157 .u.insns = {
6158 BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
6159 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, SKF_LL_OFF + 0x3e),
6160 BPF_STMT(BPF_MISC | BPF_TXA, 0),
6161 BPF_STMT(BPF_RET | BPF_A, 0x0),
6162 },
6163 CLASSIC,
6164 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6165 { {0x40, 0x24 }, },
6166 },
6167 {
6168 "LDX_MSH standalone, out of bounds",
6169 .u.insns = {
6170 BPF_STMT(BPF_LD | BPF_IMM, 0xffeebbaa),
6171 BPF_STMT(BPF_LDX | BPF_B | BPF_MSH, 0x40),
6172 BPF_STMT(BPF_MISC | BPF_TXA, 0),
6173 BPF_STMT(BPF_RET | BPF_A, 0x0),
6174 },
6175 CLASSIC,
6176 { [0x3c] = 0x25, [0x3d] = 0x05, [0x3e] = 0x19, [0x3f] = 0x82 },
6177 { {0x40, 0 }, },
6178 },
5942 /* 6179 /*
5943 * verify that the interpreter or JIT correctly sets A and X 6180 * verify that the interpreter or JIT correctly sets A and X
5944 * to 0. 6181 * to 0.
@@ -6127,14 +6364,6 @@ static struct bpf_test tests[] = {
6127 {}, 6364 {},
6128 { {0x1, 0x42 } }, 6365 { {0x1, 0x42 } },
6129 }, 6366 },
6130 {
6131 "LD_ABS with helper changing skb data",
6132 { },
6133 INTERNAL,
6134 { 0x34 },
6135 { { ETH_HLEN, 42 } },
6136 .fill_helper = bpf_fill_ld_abs_vlan_push_pop2,
6137 },
6138 /* Checking interpreter vs JIT wrt signed extended imms. */ 6367 /* Checking interpreter vs JIT wrt signed extended imms. */
6139 { 6368 {
6140 "JNE signed compare, test 1", 6369 "JNE signed compare, test 1",
diff --git a/lib/test_firmware.c b/lib/test_firmware.c
index cee000ac54d8..b984806d7d7b 100644
--- a/lib/test_firmware.c
+++ b/lib/test_firmware.c
@@ -618,8 +618,9 @@ static ssize_t trigger_batched_requests_store(struct device *dev,
618 618
619 mutex_lock(&test_fw_mutex); 619 mutex_lock(&test_fw_mutex);
620 620
621 test_fw_config->reqs = vzalloc(sizeof(struct test_batched_req) * 621 test_fw_config->reqs =
622 test_fw_config->num_requests * 2); 622 vzalloc(array3_size(sizeof(struct test_batched_req),
623 test_fw_config->num_requests, 2));
623 if (!test_fw_config->reqs) { 624 if (!test_fw_config->reqs) {
624 rc = -ENOMEM; 625 rc = -ENOMEM;
625 goto out_unlock; 626 goto out_unlock;
@@ -720,8 +721,9 @@ ssize_t trigger_batched_requests_async_store(struct device *dev,
720 721
721 mutex_lock(&test_fw_mutex); 722 mutex_lock(&test_fw_mutex);
722 723
723 test_fw_config->reqs = vzalloc(sizeof(struct test_batched_req) * 724 test_fw_config->reqs =
724 test_fw_config->num_requests * 2); 725 vzalloc(array3_size(sizeof(struct test_batched_req),
726 test_fw_config->num_requests, 2));
725 if (!test_fw_config->reqs) { 727 if (!test_fw_config->reqs) {
726 rc = -ENOMEM; 728 rc = -ENOMEM;
727 goto out; 729 goto out;
diff --git a/lib/test_kmod.c b/lib/test_kmod.c
index 0e5b7a61460b..e3ddd836491f 100644
--- a/lib/test_kmod.c
+++ b/lib/test_kmod.c
@@ -779,8 +779,9 @@ static int kmod_config_sync_info(struct kmod_test_device *test_dev)
779 struct test_config *config = &test_dev->config; 779 struct test_config *config = &test_dev->config;
780 780
781 free_test_dev_info(test_dev); 781 free_test_dev_info(test_dev);
782 test_dev->info = vzalloc(config->num_threads * 782 test_dev->info =
783 sizeof(struct kmod_test_device_info)); 783 vzalloc(array_size(sizeof(struct kmod_test_device_info),
784 config->num_threads));
784 if (!test_dev->info) 785 if (!test_dev->info)
785 return -ENOMEM; 786 return -ENOMEM;
786 787
diff --git a/lib/test_overflow.c b/lib/test_overflow.c
new file mode 100644
index 000000000000..2278fe05a1b0
--- /dev/null
+++ b/lib/test_overflow.c
@@ -0,0 +1,417 @@
1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Test cases for arithmetic overflow checks.
4 */
5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6
7#include <linux/device.h>
8#include <linux/init.h>
9#include <linux/kernel.h>
10#include <linux/mm.h>
11#include <linux/module.h>
12#include <linux/overflow.h>
13#include <linux/slab.h>
14#include <linux/types.h>
15#include <linux/vmalloc.h>
16
17#define DEFINE_TEST_ARRAY(t) \
18 static const struct test_ ## t { \
19 t a, b; \
20 t sum, diff, prod; \
21 bool s_of, d_of, p_of; \
22 } t ## _tests[] __initconst
23
24DEFINE_TEST_ARRAY(u8) = {
25 {0, 0, 0, 0, 0, false, false, false},
26 {1, 1, 2, 0, 1, false, false, false},
27 {0, 1, 1, U8_MAX, 0, false, true, false},
28 {1, 0, 1, 1, 0, false, false, false},
29 {0, U8_MAX, U8_MAX, 1, 0, false, true, false},
30 {U8_MAX, 0, U8_MAX, U8_MAX, 0, false, false, false},
31 {1, U8_MAX, 0, 2, U8_MAX, true, true, false},
32 {U8_MAX, 1, 0, U8_MAX-1, U8_MAX, true, false, false},
33 {U8_MAX, U8_MAX, U8_MAX-1, 0, 1, true, false, true},
34
35 {U8_MAX, U8_MAX-1, U8_MAX-2, 1, 2, true, false, true},
36 {U8_MAX-1, U8_MAX, U8_MAX-2, U8_MAX, 2, true, true, true},
37
38 {1U << 3, 1U << 3, 1U << 4, 0, 1U << 6, false, false, false},
39 {1U << 4, 1U << 4, 1U << 5, 0, 0, false, false, true},
40 {1U << 4, 1U << 3, 3*(1U << 3), 1U << 3, 1U << 7, false, false, false},
41 {1U << 7, 1U << 7, 0, 0, 0, true, false, true},
42
43 {48, 32, 80, 16, 0, false, false, true},
44 {128, 128, 0, 0, 0, true, false, true},
45 {123, 234, 101, 145, 110, true, true, true},
46};
47DEFINE_TEST_ARRAY(u16) = {
48 {0, 0, 0, 0, 0, false, false, false},
49 {1, 1, 2, 0, 1, false, false, false},
50 {0, 1, 1, U16_MAX, 0, false, true, false},
51 {1, 0, 1, 1, 0, false, false, false},
52 {0, U16_MAX, U16_MAX, 1, 0, false, true, false},
53 {U16_MAX, 0, U16_MAX, U16_MAX, 0, false, false, false},
54 {1, U16_MAX, 0, 2, U16_MAX, true, true, false},
55 {U16_MAX, 1, 0, U16_MAX-1, U16_MAX, true, false, false},
56 {U16_MAX, U16_MAX, U16_MAX-1, 0, 1, true, false, true},
57
58 {U16_MAX, U16_MAX-1, U16_MAX-2, 1, 2, true, false, true},
59 {U16_MAX-1, U16_MAX, U16_MAX-2, U16_MAX, 2, true, true, true},
60
61 {1U << 7, 1U << 7, 1U << 8, 0, 1U << 14, false, false, false},
62 {1U << 8, 1U << 8, 1U << 9, 0, 0, false, false, true},
63 {1U << 8, 1U << 7, 3*(1U << 7), 1U << 7, 1U << 15, false, false, false},
64 {1U << 15, 1U << 15, 0, 0, 0, true, false, true},
65
66 {123, 234, 357, 65425, 28782, false, true, false},
67 {1234, 2345, 3579, 64425, 10146, false, true, true},
68};
69DEFINE_TEST_ARRAY(u32) = {
70 {0, 0, 0, 0, 0, false, false, false},
71 {1, 1, 2, 0, 1, false, false, false},
72 {0, 1, 1, U32_MAX, 0, false, true, false},
73 {1, 0, 1, 1, 0, false, false, false},
74 {0, U32_MAX, U32_MAX, 1, 0, false, true, false},
75 {U32_MAX, 0, U32_MAX, U32_MAX, 0, false, false, false},
76 {1, U32_MAX, 0, 2, U32_MAX, true, true, false},
77 {U32_MAX, 1, 0, U32_MAX-1, U32_MAX, true, false, false},
78 {U32_MAX, U32_MAX, U32_MAX-1, 0, 1, true, false, true},
79
80 {U32_MAX, U32_MAX-1, U32_MAX-2, 1, 2, true, false, true},
81 {U32_MAX-1, U32_MAX, U32_MAX-2, U32_MAX, 2, true, true, true},
82
83 {1U << 15, 1U << 15, 1U << 16, 0, 1U << 30, false, false, false},
84 {1U << 16, 1U << 16, 1U << 17, 0, 0, false, false, true},
85 {1U << 16, 1U << 15, 3*(1U << 15), 1U << 15, 1U << 31, false, false, false},
86 {1U << 31, 1U << 31, 0, 0, 0, true, false, true},
87
88 {-2U, 1U, -1U, -3U, -2U, false, false, false},
89 {-4U, 5U, 1U, -9U, -20U, true, false, true},
90};
91
92DEFINE_TEST_ARRAY(u64) = {
93 {0, 0, 0, 0, 0, false, false, false},
94 {1, 1, 2, 0, 1, false, false, false},
95 {0, 1, 1, U64_MAX, 0, false, true, false},
96 {1, 0, 1, 1, 0, false, false, false},
97 {0, U64_MAX, U64_MAX, 1, 0, false, true, false},
98 {U64_MAX, 0, U64_MAX, U64_MAX, 0, false, false, false},
99 {1, U64_MAX, 0, 2, U64_MAX, true, true, false},
100 {U64_MAX, 1, 0, U64_MAX-1, U64_MAX, true, false, false},
101 {U64_MAX, U64_MAX, U64_MAX-1, 0, 1, true, false, true},
102
103 {U64_MAX, U64_MAX-1, U64_MAX-2, 1, 2, true, false, true},
104 {U64_MAX-1, U64_MAX, U64_MAX-2, U64_MAX, 2, true, true, true},
105
106 {1ULL << 31, 1ULL << 31, 1ULL << 32, 0, 1ULL << 62, false, false, false},
107 {1ULL << 32, 1ULL << 32, 1ULL << 33, 0, 0, false, false, true},
108 {1ULL << 32, 1ULL << 31, 3*(1ULL << 31), 1ULL << 31, 1ULL << 63, false, false, false},
109 {1ULL << 63, 1ULL << 63, 0, 0, 0, true, false, true},
110 {1000000000ULL /* 10^9 */, 10000000000ULL /* 10^10 */,
111 11000000000ULL, 18446744064709551616ULL, 10000000000000000000ULL,
112 false, true, false},
113 {-15ULL, 10ULL, -5ULL, -25ULL, -150ULL, false, false, true},
114};
115
116DEFINE_TEST_ARRAY(s8) = {
117 {0, 0, 0, 0, 0, false, false, false},
118
119 {0, S8_MAX, S8_MAX, -S8_MAX, 0, false, false, false},
120 {S8_MAX, 0, S8_MAX, S8_MAX, 0, false, false, false},
121 {0, S8_MIN, S8_MIN, S8_MIN, 0, false, true, false},
122 {S8_MIN, 0, S8_MIN, S8_MIN, 0, false, false, false},
123
124 {-1, S8_MIN, S8_MAX, S8_MAX, S8_MIN, true, false, true},
125 {S8_MIN, -1, S8_MAX, -S8_MAX, S8_MIN, true, false, true},
126 {-1, S8_MAX, S8_MAX-1, S8_MIN, -S8_MAX, false, false, false},
127 {S8_MAX, -1, S8_MAX-1, S8_MIN, -S8_MAX, false, true, false},
128 {-1, -S8_MAX, S8_MIN, S8_MAX-1, S8_MAX, false, false, false},
129 {-S8_MAX, -1, S8_MIN, S8_MIN+2, S8_MAX, false, false, false},
130
131 {1, S8_MIN, -S8_MAX, -S8_MAX, S8_MIN, false, true, false},
132 {S8_MIN, 1, -S8_MAX, S8_MAX, S8_MIN, false, true, false},
133 {1, S8_MAX, S8_MIN, S8_MIN+2, S8_MAX, true, false, false},
134 {S8_MAX, 1, S8_MIN, S8_MAX-1, S8_MAX, true, false, false},
135
136 {S8_MIN, S8_MIN, 0, 0, 0, true, false, true},
137 {S8_MAX, S8_MAX, -2, 0, 1, true, false, true},
138
139 {-4, -32, -36, 28, -128, false, false, true},
140 {-4, 32, 28, -36, -128, false, false, false},
141};
142
143DEFINE_TEST_ARRAY(s16) = {
144 {0, 0, 0, 0, 0, false, false, false},
145
146 {0, S16_MAX, S16_MAX, -S16_MAX, 0, false, false, false},
147 {S16_MAX, 0, S16_MAX, S16_MAX, 0, false, false, false},
148 {0, S16_MIN, S16_MIN, S16_MIN, 0, false, true, false},
149 {S16_MIN, 0, S16_MIN, S16_MIN, 0, false, false, false},
150
151 {-1, S16_MIN, S16_MAX, S16_MAX, S16_MIN, true, false, true},
152 {S16_MIN, -1, S16_MAX, -S16_MAX, S16_MIN, true, false, true},
153 {-1, S16_MAX, S16_MAX-1, S16_MIN, -S16_MAX, false, false, false},
154 {S16_MAX, -1, S16_MAX-1, S16_MIN, -S16_MAX, false, true, false},
155 {-1, -S16_MAX, S16_MIN, S16_MAX-1, S16_MAX, false, false, false},
156 {-S16_MAX, -1, S16_MIN, S16_MIN+2, S16_MAX, false, false, false},
157
158 {1, S16_MIN, -S16_MAX, -S16_MAX, S16_MIN, false, true, false},
159 {S16_MIN, 1, -S16_MAX, S16_MAX, S16_MIN, false, true, false},
160 {1, S16_MAX, S16_MIN, S16_MIN+2, S16_MAX, true, false, false},
161 {S16_MAX, 1, S16_MIN, S16_MAX-1, S16_MAX, true, false, false},
162
163 {S16_MIN, S16_MIN, 0, 0, 0, true, false, true},
164 {S16_MAX, S16_MAX, -2, 0, 1, true, false, true},
165};
166DEFINE_TEST_ARRAY(s32) = {
167 {0, 0, 0, 0, 0, false, false, false},
168
169 {0, S32_MAX, S32_MAX, -S32_MAX, 0, false, false, false},
170 {S32_MAX, 0, S32_MAX, S32_MAX, 0, false, false, false},
171 {0, S32_MIN, S32_MIN, S32_MIN, 0, false, true, false},
172 {S32_MIN, 0, S32_MIN, S32_MIN, 0, false, false, false},
173
174 {-1, S32_MIN, S32_MAX, S32_MAX, S32_MIN, true, false, true},
175 {S32_MIN, -1, S32_MAX, -S32_MAX, S32_MIN, true, false, true},
176 {-1, S32_MAX, S32_MAX-1, S32_MIN, -S32_MAX, false, false, false},
177 {S32_MAX, -1, S32_MAX-1, S32_MIN, -S32_MAX, false, true, false},
178 {-1, -S32_MAX, S32_MIN, S32_MAX-1, S32_MAX, false, false, false},
179 {-S32_MAX, -1, S32_MIN, S32_MIN+2, S32_MAX, false, false, false},
180
181 {1, S32_MIN, -S32_MAX, -S32_MAX, S32_MIN, false, true, false},
182 {S32_MIN, 1, -S32_MAX, S32_MAX, S32_MIN, false, true, false},
183 {1, S32_MAX, S32_MIN, S32_MIN+2, S32_MAX, true, false, false},
184 {S32_MAX, 1, S32_MIN, S32_MAX-1, S32_MAX, true, false, false},
185
186 {S32_MIN, S32_MIN, 0, 0, 0, true, false, true},
187 {S32_MAX, S32_MAX, -2, 0, 1, true, false, true},
188};
189DEFINE_TEST_ARRAY(s64) = {
190 {0, 0, 0, 0, 0, false, false, false},
191
192 {0, S64_MAX, S64_MAX, -S64_MAX, 0, false, false, false},
193 {S64_MAX, 0, S64_MAX, S64_MAX, 0, false, false, false},
194 {0, S64_MIN, S64_MIN, S64_MIN, 0, false, true, false},
195 {S64_MIN, 0, S64_MIN, S64_MIN, 0, false, false, false},
196
197 {-1, S64_MIN, S64_MAX, S64_MAX, S64_MIN, true, false, true},
198 {S64_MIN, -1, S64_MAX, -S64_MAX, S64_MIN, true, false, true},
199 {-1, S64_MAX, S64_MAX-1, S64_MIN, -S64_MAX, false, false, false},
200 {S64_MAX, -1, S64_MAX-1, S64_MIN, -S64_MAX, false, true, false},
201 {-1, -S64_MAX, S64_MIN, S64_MAX-1, S64_MAX, false, false, false},
202 {-S64_MAX, -1, S64_MIN, S64_MIN+2, S64_MAX, false, false, false},
203
204 {1, S64_MIN, -S64_MAX, -S64_MAX, S64_MIN, false, true, false},
205 {S64_MIN, 1, -S64_MAX, S64_MAX, S64_MIN, false, true, false},
206 {1, S64_MAX, S64_MIN, S64_MIN+2, S64_MAX, true, false, false},
207 {S64_MAX, 1, S64_MIN, S64_MAX-1, S64_MAX, true, false, false},
208
209 {S64_MIN, S64_MIN, 0, 0, 0, true, false, true},
210 {S64_MAX, S64_MAX, -2, 0, 1, true, false, true},
211
212 {-1, -1, -2, 0, 1, false, false, false},
213 {-1, -128, -129, 127, 128, false, false, false},
214 {-128, -1, -129, -127, 128, false, false, false},
215 {0, -S64_MAX, -S64_MAX, S64_MAX, 0, false, false, false},
216};
217
218#define check_one_op(t, fmt, op, sym, a, b, r, of) do { \
219 t _r; \
220 bool _of; \
221 \
222 _of = check_ ## op ## _overflow(a, b, &_r); \
223 if (_of != of) { \
224 pr_warn("expected "fmt" "sym" "fmt \
225 " to%s overflow (type %s)\n", \
226 a, b, of ? "" : " not", #t); \
227 err = 1; \
228 } \
229 if (_r != r) { \
230 pr_warn("expected "fmt" "sym" "fmt" == " \
231 fmt", got "fmt" (type %s)\n", \
232 a, b, r, _r, #t); \
233 err = 1; \
234 } \
235} while (0)
236
237#define DEFINE_TEST_FUNC(t, fmt) \
238static int __init do_test_ ## t(const struct test_ ## t *p) \
239{ \
240 int err = 0; \
241 \
242 check_one_op(t, fmt, add, "+", p->a, p->b, p->sum, p->s_of); \
243 check_one_op(t, fmt, add, "+", p->b, p->a, p->sum, p->s_of); \
244 check_one_op(t, fmt, sub, "-", p->a, p->b, p->diff, p->d_of); \
245 check_one_op(t, fmt, mul, "*", p->a, p->b, p->prod, p->p_of); \
246 check_one_op(t, fmt, mul, "*", p->b, p->a, p->prod, p->p_of); \
247 \
248 return err; \
249} \
250 \
251static int __init test_ ## t ## _overflow(void) { \
252 int err = 0; \
253 unsigned i; \
254 \
255 pr_info("%-3s: %zu tests\n", #t, ARRAY_SIZE(t ## _tests)); \
256 for (i = 0; i < ARRAY_SIZE(t ## _tests); ++i) \
257 err |= do_test_ ## t(&t ## _tests[i]); \
258 return err; \
259}
260
261DEFINE_TEST_FUNC(u8, "%d");
262DEFINE_TEST_FUNC(s8, "%d");
263DEFINE_TEST_FUNC(u16, "%d");
264DEFINE_TEST_FUNC(s16, "%d");
265DEFINE_TEST_FUNC(u32, "%u");
266DEFINE_TEST_FUNC(s32, "%d");
267#if BITS_PER_LONG == 64
268DEFINE_TEST_FUNC(u64, "%llu");
269DEFINE_TEST_FUNC(s64, "%lld");
270#endif
271
272static int __init test_overflow_calculation(void)
273{
274 int err = 0;
275
276 err |= test_u8_overflow();
277 err |= test_s8_overflow();
278 err |= test_u16_overflow();
279 err |= test_s16_overflow();
280 err |= test_u32_overflow();
281 err |= test_s32_overflow();
282#if BITS_PER_LONG == 64
283 err |= test_u64_overflow();
284 err |= test_s64_overflow();
285#endif
286
287 return err;
288}
289
290/*
291 * Deal with the various forms of allocator arguments. See comments above
292 * the DEFINE_TEST_ALLOC() instances for mapping of the "bits".
293 */
294#define alloc010(alloc, arg, sz) alloc(sz, GFP_KERNEL)
295#define alloc011(alloc, arg, sz) alloc(sz, GFP_KERNEL, NUMA_NO_NODE)
296#define alloc000(alloc, arg, sz) alloc(sz)
297#define alloc001(alloc, arg, sz) alloc(sz, NUMA_NO_NODE)
298#define alloc110(alloc, arg, sz) alloc(arg, sz, GFP_KERNEL)
299#define free0(free, arg, ptr) free(ptr)
300#define free1(free, arg, ptr) free(arg, ptr)
301
302/* Wrap around to 8K */
303#define TEST_SIZE (9 << PAGE_SHIFT)
304
305#define DEFINE_TEST_ALLOC(func, free_func, want_arg, want_gfp, want_node)\
306static int __init test_ ## func (void *arg) \
307{ \
308 volatile size_t a = TEST_SIZE; \
309 volatile size_t b = (SIZE_MAX / TEST_SIZE) + 1; \
310 void *ptr; \
311 \
312 /* Tiny allocation test. */ \
313 ptr = alloc ## want_arg ## want_gfp ## want_node (func, arg, 1);\
314 if (!ptr) { \
315 pr_warn(#func " failed regular allocation?!\n"); \
316 return 1; \
317 } \
318 free ## want_arg (free_func, arg, ptr); \
319 \
320 /* Wrapped allocation test. */ \
321 ptr = alloc ## want_arg ## want_gfp ## want_node (func, arg, \
322 a * b); \
323 if (!ptr) { \
324 pr_warn(#func " unexpectedly failed bad wrapping?!\n"); \
325 return 1; \
326 } \
327 free ## want_arg (free_func, arg, ptr); \
328 \
329 /* Saturated allocation test. */ \
330 ptr = alloc ## want_arg ## want_gfp ## want_node (func, arg, \
331 array_size(a, b)); \
332 if (ptr) { \
333 pr_warn(#func " missed saturation!\n"); \
334 free ## want_arg (free_func, arg, ptr); \
335 return 1; \
336 } \
337 pr_info(#func " detected saturation\n"); \
338 return 0; \
339}
340
341/*
342 * Allocator uses a trailing node argument --------+ (e.g. kmalloc_node())
343 * Allocator uses the gfp_t argument -----------+ | (e.g. kmalloc())
344 * Allocator uses a special leading argument + | | (e.g. devm_kmalloc())
345 * | | |
346 */
347DEFINE_TEST_ALLOC(kmalloc, kfree, 0, 1, 0);
348DEFINE_TEST_ALLOC(kmalloc_node, kfree, 0, 1, 1);
349DEFINE_TEST_ALLOC(kzalloc, kfree, 0, 1, 0);
350DEFINE_TEST_ALLOC(kzalloc_node, kfree, 0, 1, 1);
351DEFINE_TEST_ALLOC(vmalloc, vfree, 0, 0, 0);
352DEFINE_TEST_ALLOC(vmalloc_node, vfree, 0, 0, 1);
353DEFINE_TEST_ALLOC(vzalloc, vfree, 0, 0, 0);
354DEFINE_TEST_ALLOC(vzalloc_node, vfree, 0, 0, 1);
355DEFINE_TEST_ALLOC(kvmalloc, kvfree, 0, 1, 0);
356DEFINE_TEST_ALLOC(kvmalloc_node, kvfree, 0, 1, 1);
357DEFINE_TEST_ALLOC(kvzalloc, kvfree, 0, 1, 0);
358DEFINE_TEST_ALLOC(kvzalloc_node, kvfree, 0, 1, 1);
359DEFINE_TEST_ALLOC(devm_kmalloc, devm_kfree, 1, 1, 0);
360DEFINE_TEST_ALLOC(devm_kzalloc, devm_kfree, 1, 1, 0);
361
362static int __init test_overflow_allocation(void)
363{
364 const char device_name[] = "overflow-test";
365 struct device *dev;
366 int err = 0;
367
368 /* Create dummy device for devm_kmalloc()-family tests. */
369 dev = root_device_register(device_name);
370 if (IS_ERR(dev)) {
371 pr_warn("Cannot register test device\n");
372 return 1;
373 }
374
375 err |= test_kmalloc(NULL);
376 err |= test_kmalloc_node(NULL);
377 err |= test_kzalloc(NULL);
378 err |= test_kzalloc_node(NULL);
379 err |= test_kvmalloc(NULL);
380 err |= test_kvmalloc_node(NULL);
381 err |= test_kvzalloc(NULL);
382 err |= test_kvzalloc_node(NULL);
383 err |= test_vmalloc(NULL);
384 err |= test_vmalloc_node(NULL);
385 err |= test_vzalloc(NULL);
386 err |= test_vzalloc_node(NULL);
387 err |= test_devm_kmalloc(dev);
388 err |= test_devm_kzalloc(dev);
389
390 device_unregister(dev);
391
392 return err;
393}
394
395static int __init test_module_init(void)
396{
397 int err = 0;
398
399 err |= test_overflow_calculation();
400 err |= test_overflow_allocation();
401
402 if (err) {
403 pr_warn("FAIL!\n");
404 err = -EINVAL;
405 } else {
406 pr_info("all tests passed\n");
407 }
408
409 return err;
410}
411
412static void __exit test_module_exit(void)
413{ }
414
415module_init(test_module_init);
416module_exit(test_module_exit);
417MODULE_LICENSE("Dual MIT/GPL");
diff --git a/lib/test_printf.c b/lib/test_printf.c
index 71ebfa43ad05..cea592f402ed 100644
--- a/lib/test_printf.c
+++ b/lib/test_printf.c
@@ -204,7 +204,7 @@ test_string(void)
204#if BITS_PER_LONG == 64 204#if BITS_PER_LONG == 64
205 205
206#define PTR_WIDTH 16 206#define PTR_WIDTH 16
207#define PTR ((void *)0xffff0123456789ab) 207#define PTR ((void *)0xffff0123456789abUL)
208#define PTR_STR "ffff0123456789ab" 208#define PTR_STR "ffff0123456789ab"
209#define ZEROS "00000000" /* hex 32 zero bits */ 209#define ZEROS "00000000" /* hex 32 zero bits */
210 210
diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c
index f4000c137dbe..fb6968109113 100644
--- a/lib/test_rhashtable.c
+++ b/lib/test_rhashtable.c
@@ -285,12 +285,14 @@ static int __init test_rhltable(unsigned int entries)
285 if (entries == 0) 285 if (entries == 0)
286 entries = 1; 286 entries = 1;
287 287
288 rhl_test_objects = vzalloc(sizeof(*rhl_test_objects) * entries); 288 rhl_test_objects = vzalloc(array_size(entries,
289 sizeof(*rhl_test_objects)));
289 if (!rhl_test_objects) 290 if (!rhl_test_objects)
290 return -ENOMEM; 291 return -ENOMEM;
291 292
292 ret = -ENOMEM; 293 ret = -ENOMEM;
293 obj_in_table = vzalloc(BITS_TO_LONGS(entries) * sizeof(unsigned long)); 294 obj_in_table = vzalloc(array_size(sizeof(unsigned long),
295 BITS_TO_LONGS(entries)));
294 if (!obj_in_table) 296 if (!obj_in_table)
295 goto out_free; 297 goto out_free;
296 298
@@ -706,7 +708,8 @@ static int __init test_rht_init(void)
706 test_rht_params.max_size = max_size ? : roundup_pow_of_two(entries); 708 test_rht_params.max_size = max_size ? : roundup_pow_of_two(entries);
707 test_rht_params.nelem_hint = size; 709 test_rht_params.nelem_hint = size;
708 710
709 objs = vzalloc((test_rht_params.max_size + 1) * sizeof(struct test_obj)); 711 objs = vzalloc(array_size(sizeof(struct test_obj),
712 test_rht_params.max_size + 1));
710 if (!objs) 713 if (!objs)
711 return -ENOMEM; 714 return -ENOMEM;
712 715
@@ -753,10 +756,10 @@ static int __init test_rht_init(void)
753 pr_info("Testing concurrent rhashtable access from %d threads\n", 756 pr_info("Testing concurrent rhashtable access from %d threads\n",
754 tcount); 757 tcount);
755 sema_init(&prestart_sem, 1 - tcount); 758 sema_init(&prestart_sem, 1 - tcount);
756 tdata = vzalloc(tcount * sizeof(struct thread_data)); 759 tdata = vzalloc(array_size(tcount, sizeof(struct thread_data)));
757 if (!tdata) 760 if (!tdata)
758 return -ENOMEM; 761 return -ENOMEM;
759 objs = vzalloc(tcount * entries * sizeof(struct test_obj)); 762 objs = vzalloc(array3_size(sizeof(struct test_obj), tcount, entries));
760 if (!objs) { 763 if (!objs) {
761 vfree(tdata); 764 vfree(tdata);
762 return -ENOMEM; 765 return -ENOMEM;
diff --git a/lib/ucmpdi2.c b/lib/ucmpdi2.c
index 25ca2d4c1e19..597998169a96 100644
--- a/lib/ucmpdi2.c
+++ b/lib/ucmpdi2.c
@@ -17,7 +17,7 @@
17#include <linux/module.h> 17#include <linux/module.h>
18#include <linux/libgcc.h> 18#include <linux/libgcc.h>
19 19
20word_type __ucmpdi2(unsigned long long a, unsigned long long b) 20word_type notrace __ucmpdi2(unsigned long long a, unsigned long long b)
21{ 21{
22 const DWunion au = {.ll = a}; 22 const DWunion au = {.ll = a};
23 const DWunion bu = {.ll = b}; 23 const DWunion bu = {.ll = b};
diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c
index d7e06b28de38..0a559a42359b 100644
--- a/lib/ucs2_string.c
+++ b/lib/ucs2_string.c
@@ -112,3 +112,5 @@ ucs2_as_utf8(u8 *dest, const ucs2_char_t *src, unsigned long maxlength)
112 return j; 112 return j;
113} 113}
114EXPORT_SYMBOL(ucs2_as_utf8); 114EXPORT_SYMBOL(ucs2_as_utf8);
115
116MODULE_LICENSE("GPL v2");
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 30c0cb8cc9bc..a48aaa79d352 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -703,6 +703,22 @@ char *symbol_string(char *buf, char *end, void *ptr,
703#endif 703#endif
704} 704}
705 705
706static const struct printf_spec default_str_spec = {
707 .field_width = -1,
708 .precision = -1,
709};
710
711static const struct printf_spec default_flag_spec = {
712 .base = 16,
713 .precision = -1,
714 .flags = SPECIAL | SMALL,
715};
716
717static const struct printf_spec default_dec_spec = {
718 .base = 10,
719 .precision = -1,
720};
721
706static noinline_for_stack 722static noinline_for_stack
707char *resource_string(char *buf, char *end, struct resource *res, 723char *resource_string(char *buf, char *end, struct resource *res,
708 struct printf_spec spec, const char *fmt) 724 struct printf_spec spec, const char *fmt)
@@ -732,21 +748,11 @@ char *resource_string(char *buf, char *end, struct resource *res,
732 .precision = -1, 748 .precision = -1,
733 .flags = SMALL | ZEROPAD, 749 .flags = SMALL | ZEROPAD,
734 }; 750 };
735 static const struct printf_spec dec_spec = {
736 .base = 10,
737 .precision = -1,
738 .flags = 0,
739 };
740 static const struct printf_spec str_spec = { 751 static const struct printf_spec str_spec = {
741 .field_width = -1, 752 .field_width = -1,
742 .precision = 10, 753 .precision = 10,
743 .flags = LEFT, 754 .flags = LEFT,
744 }; 755 };
745 static const struct printf_spec flag_spec = {
746 .base = 16,
747 .precision = -1,
748 .flags = SPECIAL | SMALL,
749 };
750 756
751 /* 32-bit res (sizeof==4): 10 chars in dec, 10 in hex ("0x" + 8) 757 /* 32-bit res (sizeof==4): 10 chars in dec, 10 in hex ("0x" + 8)
752 * 64-bit res (sizeof==8): 20 chars in dec, 18 in hex ("0x" + 16) */ 758 * 64-bit res (sizeof==8): 20 chars in dec, 18 in hex ("0x" + 16) */
@@ -770,10 +776,10 @@ char *resource_string(char *buf, char *end, struct resource *res,
770 specp = &mem_spec; 776 specp = &mem_spec;
771 } else if (res->flags & IORESOURCE_IRQ) { 777 } else if (res->flags & IORESOURCE_IRQ) {
772 p = string(p, pend, "irq ", str_spec); 778 p = string(p, pend, "irq ", str_spec);
773 specp = &dec_spec; 779 specp = &default_dec_spec;
774 } else if (res->flags & IORESOURCE_DMA) { 780 } else if (res->flags & IORESOURCE_DMA) {
775 p = string(p, pend, "dma ", str_spec); 781 p = string(p, pend, "dma ", str_spec);
776 specp = &dec_spec; 782 specp = &default_dec_spec;
777 } else if (res->flags & IORESOURCE_BUS) { 783 } else if (res->flags & IORESOURCE_BUS) {
778 p = string(p, pend, "bus ", str_spec); 784 p = string(p, pend, "bus ", str_spec);
779 specp = &bus_spec; 785 specp = &bus_spec;
@@ -803,7 +809,7 @@ char *resource_string(char *buf, char *end, struct resource *res,
803 p = string(p, pend, " disabled", str_spec); 809 p = string(p, pend, " disabled", str_spec);
804 } else { 810 } else {
805 p = string(p, pend, " flags ", str_spec); 811 p = string(p, pend, " flags ", str_spec);
806 p = number(p, pend, res->flags, flag_spec); 812 p = number(p, pend, res->flags, default_flag_spec);
807 } 813 }
808 *p++ = ']'; 814 *p++ = ']';
809 *p = '\0'; 815 *p = '\0';
@@ -913,9 +919,6 @@ char *bitmap_list_string(char *buf, char *end, unsigned long *bitmap,
913 int cur, rbot, rtop; 919 int cur, rbot, rtop;
914 bool first = true; 920 bool first = true;
915 921
916 /* reused to print numbers */
917 spec = (struct printf_spec){ .base = 10 };
918
919 rbot = cur = find_first_bit(bitmap, nr_bits); 922 rbot = cur = find_first_bit(bitmap, nr_bits);
920 while (cur < nr_bits) { 923 while (cur < nr_bits) {
921 rtop = cur; 924 rtop = cur;
@@ -930,13 +933,13 @@ char *bitmap_list_string(char *buf, char *end, unsigned long *bitmap,
930 } 933 }
931 first = false; 934 first = false;
932 935
933 buf = number(buf, end, rbot, spec); 936 buf = number(buf, end, rbot, default_dec_spec);
934 if (rbot < rtop) { 937 if (rbot < rtop) {
935 if (buf < end) 938 if (buf < end)
936 *buf = '-'; 939 *buf = '-';
937 buf++; 940 buf++;
938 941
939 buf = number(buf, end, rtop, spec); 942 buf = number(buf, end, rtop, default_dec_spec);
940 } 943 }
941 944
942 rbot = cur; 945 rbot = cur;
@@ -1354,11 +1357,9 @@ char *uuid_string(char *buf, char *end, const u8 *addr,
1354 return string(buf, end, uuid, spec); 1357 return string(buf, end, uuid, spec);
1355} 1358}
1356 1359
1357int kptr_restrict __read_mostly;
1358
1359static noinline_for_stack 1360static noinline_for_stack
1360char *restricted_pointer(char *buf, char *end, const void *ptr, 1361char *pointer_string(char *buf, char *end, const void *ptr,
1361 struct printf_spec spec) 1362 struct printf_spec spec)
1362{ 1363{
1363 spec.base = 16; 1364 spec.base = 16;
1364 spec.flags |= SMALL; 1365 spec.flags |= SMALL;
@@ -1367,6 +1368,15 @@ char *restricted_pointer(char *buf, char *end, const void *ptr,
1367 spec.flags |= ZEROPAD; 1368 spec.flags |= ZEROPAD;
1368 } 1369 }
1369 1370
1371 return number(buf, end, (unsigned long int)ptr, spec);
1372}
1373
1374int kptr_restrict __read_mostly;
1375
1376static noinline_for_stack
1377char *restricted_pointer(char *buf, char *end, const void *ptr,
1378 struct printf_spec spec)
1379{
1370 switch (kptr_restrict) { 1380 switch (kptr_restrict) {
1371 case 0: 1381 case 0:
1372 /* Always print %pK values */ 1382 /* Always print %pK values */
@@ -1378,8 +1388,11 @@ char *restricted_pointer(char *buf, char *end, const void *ptr,
1378 * kptr_restrict==1 cannot be used in IRQ context 1388 * kptr_restrict==1 cannot be used in IRQ context
1379 * because its test for CAP_SYSLOG would be meaningless. 1389 * because its test for CAP_SYSLOG would be meaningless.
1380 */ 1390 */
1381 if (in_irq() || in_serving_softirq() || in_nmi()) 1391 if (in_irq() || in_serving_softirq() || in_nmi()) {
1392 if (spec.field_width == -1)
1393 spec.field_width = 2 * sizeof(ptr);
1382 return string(buf, end, "pK-error", spec); 1394 return string(buf, end, "pK-error", spec);
1395 }
1383 1396
1384 /* 1397 /*
1385 * Only print the real pointer value if the current 1398 * Only print the real pointer value if the current
@@ -1404,7 +1417,7 @@ char *restricted_pointer(char *buf, char *end, const void *ptr,
1404 break; 1417 break;
1405 } 1418 }
1406 1419
1407 return number(buf, end, (unsigned long)ptr, spec); 1420 return pointer_string(buf, end, ptr, spec);
1408} 1421}
1409 1422
1410static noinline_for_stack 1423static noinline_for_stack
@@ -1456,9 +1469,6 @@ char *clock(char *buf, char *end, struct clk *clk, struct printf_spec spec,
1456 return string(buf, end, NULL, spec); 1469 return string(buf, end, NULL, spec);
1457 1470
1458 switch (fmt[1]) { 1471 switch (fmt[1]) {
1459 case 'r':
1460 return number(buf, end, clk_get_rate(clk), spec);
1461
1462 case 'n': 1472 case 'n':
1463 default: 1473 default:
1464#ifdef CONFIG_COMMON_CLK 1474#ifdef CONFIG_COMMON_CLK
@@ -1474,23 +1484,13 @@ char *format_flags(char *buf, char *end, unsigned long flags,
1474 const struct trace_print_flags *names) 1484 const struct trace_print_flags *names)
1475{ 1485{
1476 unsigned long mask; 1486 unsigned long mask;
1477 const struct printf_spec strspec = {
1478 .field_width = -1,
1479 .precision = -1,
1480 };
1481 const struct printf_spec numspec = {
1482 .flags = SPECIAL|SMALL,
1483 .field_width = -1,
1484 .precision = -1,
1485 .base = 16,
1486 };
1487 1487
1488 for ( ; flags && names->name; names++) { 1488 for ( ; flags && names->name; names++) {
1489 mask = names->mask; 1489 mask = names->mask;
1490 if ((flags & mask) != mask) 1490 if ((flags & mask) != mask)
1491 continue; 1491 continue;
1492 1492
1493 buf = string(buf, end, names->name, strspec); 1493 buf = string(buf, end, names->name, default_str_spec);
1494 1494
1495 flags &= ~mask; 1495 flags &= ~mask;
1496 if (flags) { 1496 if (flags) {
@@ -1501,7 +1501,7 @@ char *format_flags(char *buf, char *end, unsigned long flags,
1501 } 1501 }
1502 1502
1503 if (flags) 1503 if (flags)
1504 buf = number(buf, end, flags, numspec); 1504 buf = number(buf, end, flags, default_flag_spec);
1505 1505
1506 return buf; 1506 return buf;
1507} 1507}
@@ -1548,22 +1548,18 @@ char *device_node_gen_full_name(const struct device_node *np, char *buf, char *e
1548{ 1548{
1549 int depth; 1549 int depth;
1550 const struct device_node *parent = np->parent; 1550 const struct device_node *parent = np->parent;
1551 static const struct printf_spec strspec = {
1552 .field_width = -1,
1553 .precision = -1,
1554 };
1555 1551
1556 /* special case for root node */ 1552 /* special case for root node */
1557 if (!parent) 1553 if (!parent)
1558 return string(buf, end, "/", strspec); 1554 return string(buf, end, "/", default_str_spec);
1559 1555
1560 for (depth = 0; parent->parent; depth++) 1556 for (depth = 0; parent->parent; depth++)
1561 parent = parent->parent; 1557 parent = parent->parent;
1562 1558
1563 for ( ; depth >= 0; depth--) { 1559 for ( ; depth >= 0; depth--) {
1564 buf = string(buf, end, "/", strspec); 1560 buf = string(buf, end, "/", default_str_spec);
1565 buf = string(buf, end, device_node_name_for_depth(np, depth), 1561 buf = string(buf, end, device_node_name_for_depth(np, depth),
1566 strspec); 1562 default_str_spec);
1567 } 1563 }
1568 return buf; 1564 return buf;
1569} 1565}
@@ -1655,33 +1651,22 @@ char *device_node_string(char *buf, char *end, struct device_node *dn,
1655 return widen_string(buf, buf - buf_start, end, spec); 1651 return widen_string(buf, buf - buf_start, end, spec);
1656} 1652}
1657 1653
1658static noinline_for_stack 1654static DEFINE_STATIC_KEY_TRUE(not_filled_random_ptr_key);
1659char *pointer_string(char *buf, char *end, const void *ptr, 1655static siphash_key_t ptr_key __read_mostly;
1660 struct printf_spec spec)
1661{
1662 spec.base = 16;
1663 spec.flags |= SMALL;
1664 if (spec.field_width == -1) {
1665 spec.field_width = 2 * sizeof(ptr);
1666 spec.flags |= ZEROPAD;
1667 }
1668 1656
1669 return number(buf, end, (unsigned long int)ptr, spec); 1657static void enable_ptr_key_workfn(struct work_struct *work)
1658{
1659 get_random_bytes(&ptr_key, sizeof(ptr_key));
1660 /* Needs to run from preemptible context */
1661 static_branch_disable(&not_filled_random_ptr_key);
1670} 1662}
1671 1663
1672static bool have_filled_random_ptr_key __read_mostly; 1664static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn);
1673static siphash_key_t ptr_key __read_mostly;
1674 1665
1675static void fill_random_ptr_key(struct random_ready_callback *unused) 1666static void fill_random_ptr_key(struct random_ready_callback *unused)
1676{ 1667{
1677 get_random_bytes(&ptr_key, sizeof(ptr_key)); 1668 /* This may be in an interrupt handler. */
1678 /* 1669 queue_work(system_unbound_wq, &enable_ptr_key_work);
1679 * have_filled_random_ptr_key==true is dependent on get_random_bytes().
1680 * ptr_to_id() needs to see have_filled_random_ptr_key==true
1681 * after get_random_bytes() returns.
1682 */
1683 smp_mb();
1684 WRITE_ONCE(have_filled_random_ptr_key, true);
1685} 1670}
1686 1671
1687static struct random_ready_callback random_ready = { 1672static struct random_ready_callback random_ready = {
@@ -1695,7 +1680,8 @@ static int __init initialize_ptr_random(void)
1695 if (!ret) { 1680 if (!ret) {
1696 return 0; 1681 return 0;
1697 } else if (ret == -EALREADY) { 1682 } else if (ret == -EALREADY) {
1698 fill_random_ptr_key(&random_ready); 1683 /* This is in preemptible context */
1684 enable_ptr_key_workfn(&enable_ptr_key_work);
1699 return 0; 1685 return 0;
1700 } 1686 }
1701 1687
@@ -1706,13 +1692,13 @@ early_initcall(initialize_ptr_random);
1706/* Maps a pointer to a 32 bit unique identifier. */ 1692/* Maps a pointer to a 32 bit unique identifier. */
1707static char *ptr_to_id(char *buf, char *end, void *ptr, struct printf_spec spec) 1693static char *ptr_to_id(char *buf, char *end, void *ptr, struct printf_spec spec)
1708{ 1694{
1695 const char *str = sizeof(ptr) == 8 ? "(____ptrval____)" : "(ptrval)";
1709 unsigned long hashval; 1696 unsigned long hashval;
1710 const int default_width = 2 * sizeof(ptr);
1711 1697
1712 if (unlikely(!have_filled_random_ptr_key)) { 1698 if (static_branch_unlikely(&not_filled_random_ptr_key)) {
1713 spec.field_width = default_width; 1699 spec.field_width = 2 * sizeof(ptr);
1714 /* string length must be less than default_width */ 1700 /* string length must be less than default_width */
1715 return string(buf, end, "(ptrval)", spec); 1701 return string(buf, end, str, spec);
1716 } 1702 }
1717 1703
1718#ifdef CONFIG_64BIT 1704#ifdef CONFIG_64BIT
@@ -1725,15 +1711,7 @@ static char *ptr_to_id(char *buf, char *end, void *ptr, struct printf_spec spec)
1725#else 1711#else
1726 hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key); 1712 hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key);
1727#endif 1713#endif
1728 1714 return pointer_string(buf, end, (const void *)hashval, spec);
1729 spec.flags |= SMALL;
1730 if (spec.field_width == -1) {
1731 spec.field_width = default_width;
1732 spec.flags |= ZEROPAD;
1733 }
1734 spec.base = 16;
1735
1736 return number(buf, end, hashval, spec);
1737} 1715}
1738 1716
1739/* 1717/*
@@ -1746,10 +1724,10 @@ static char *ptr_to_id(char *buf, char *end, void *ptr, struct printf_spec spec)
1746 * 1724 *
1747 * Right now we handle: 1725 * Right now we handle:
1748 * 1726 *
1749 * - 'F' For symbolic function descriptor pointers with offset 1727 * - 'S' For symbolic direct pointers (or function descriptors) with offset
1750 * - 'f' For simple symbolic function names without offset 1728 * - 's' For symbolic direct pointers (or function descriptors) without offset
1751 * - 'S' For symbolic direct pointers with offset 1729 * - 'F' Same as 'S'
1752 * - 's' For symbolic direct pointers without offset 1730 * - 'f' Same as 's'
1753 * - '[FfSs]R' as above with __builtin_extract_return_addr() translation 1731 * - '[FfSs]R' as above with __builtin_extract_return_addr() translation
1754 * - 'B' For backtraced symbolic direct pointers with offset 1732 * - 'B' For backtraced symbolic direct pointers with offset
1755 * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref] 1733 * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref]
@@ -1846,10 +1824,6 @@ static char *ptr_to_id(char *buf, char *end, void *ptr, struct printf_spec spec)
1846 * ** When making changes please also update: 1824 * ** When making changes please also update:
1847 * Documentation/core-api/printk-formats.rst 1825 * Documentation/core-api/printk-formats.rst
1848 * 1826 *
1849 * Note: The difference between 'S' and 'F' is that on ia64 and ppc64
1850 * function pointers are really function descriptors, which contain a
1851 * pointer to the real address.
1852 *
1853 * Note: The default behaviour (unadorned %p) is to hash the address, 1827 * Note: The default behaviour (unadorned %p) is to hash the address,
1854 * rendering it useful as a unique identifier. 1828 * rendering it useful as a unique identifier.
1855 */ 1829 */
@@ -2125,6 +2099,7 @@ qualifier:
2125 2099
2126 case 'x': 2100 case 'x':
2127 spec->flags |= SMALL; 2101 spec->flags |= SMALL;
2102 /* fall through */
2128 2103
2129 case 'X': 2104 case 'X':
2130 spec->base = 16; 2105 spec->base = 16;
@@ -3083,8 +3058,10 @@ int vsscanf(const char *buf, const char *fmt, va_list args)
3083 break; 3058 break;
3084 case 'i': 3059 case 'i':
3085 base = 0; 3060 base = 0;
3061 /* fall through */
3086 case 'd': 3062 case 'd':
3087 is_sign = true; 3063 is_sign = true;
3064 /* fall through */
3088 case 'u': 3065 case 'u':
3089 break; 3066 break;
3090 case '%': 3067 case '%':
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-01 00:07:08 -0500