aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
authorDmitry Torokhov <dmitry.torokhov@gmail.com>2013-10-08 11:43:00 -0400
committerDmitry Torokhov <dmitry.torokhov@gmail.com>2013-10-08 11:43:00 -0400
commite3c55d406bd8df1a878546002c93db90c42be10c (patch)
treeefb0ba2707c95fd7166cf1b76887c43c977e37dd /lib
parent4d6e482675f13e33599fc3d18fc723959be0a9b6 (diff)
parentd0e639c9e06d44e713170031fe05fb60ebe680af (diff)
Merge tag 'v3.12-rc4' into next
Merge with mainline to bring in changes to input subsystem that were committed through other trees.
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig12
-rw-r--r--lib/Kconfig.debug29
-rw-r--r--lib/Makefile6
-rw-r--r--lib/cpu_rmap.c6
-rw-r--r--lib/crc-t10dif.c83
-rw-r--r--lib/crc32.c17
-rw-r--r--lib/debugobjects.c20
-rw-r--r--lib/decompress_inflate.c2
-rw-r--r--lib/div64.c40
-rw-r--r--lib/dump_stack.c4
-rw-r--r--lib/dynamic_debug.c2
-rw-r--r--lib/earlycpio.c27
-rw-r--r--lib/genalloc.c22
-rw-r--r--lib/hexdump.c2
-rw-r--r--lib/kobject.c34
-rw-r--r--lib/lockref.c183
-rw-r--r--lib/lz4/lz4_decompress.c8
-rw-r--r--lib/percpu_ida.c335
-rw-r--r--lib/radix-tree.c41
-rw-r--r--lib/raid6/.gitignore1
-rw-r--r--lib/raid6/Makefile46
-rw-r--r--lib/raid6/algos.c9
-rw-r--r--lib/raid6/neon.c58
-rw-r--r--lib/raid6/neon.uc80
-rw-r--r--lib/raid6/test/Makefile35
-rw-r--r--lib/raid6/tilegx.uc86
-rw-r--r--lib/rbtree.c40
-rw-r--r--lib/rbtree_test.c12
-rw-r--r--lib/swiotlb.c8
-rw-r--r--lib/vsprintf.c82
30 files changed, 1221 insertions, 109 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index 71d9f81f6eed..b3c8be0da17f 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -48,6 +48,16 @@ config STMP_DEVICE
48config PERCPU_RWSEM 48config PERCPU_RWSEM
49 boolean 49 boolean
50 50
51config ARCH_USE_CMPXCHG_LOCKREF
52 bool
53
54config CMPXCHG_LOCKREF
55 def_bool y if ARCH_USE_CMPXCHG_LOCKREF
56 depends on SMP
57 depends on !GENERIC_LOCKBREAK
58 depends on !DEBUG_SPINLOCK
59 depends on !DEBUG_LOCK_ALLOC
60
51config CRC_CCITT 61config CRC_CCITT
52 tristate "CRC-CCITT functions" 62 tristate "CRC-CCITT functions"
53 help 63 help
@@ -66,6 +76,8 @@ config CRC16
66 76
67config CRC_T10DIF 77config CRC_T10DIF
68 tristate "CRC calculation for the T10 Data Integrity Field" 78 tristate "CRC calculation for the T10 Data Integrity Field"
79 select CRYPTO
80 select CRYPTO_CRCT10DIF
69 help 81 help
70 This option is only needed if a module that's not in the 82 This option is only needed if a module that's not in the
71 kernel tree needs to calculate CRC checks for use with the 83 kernel tree needs to calculate CRC checks for use with the
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 1501aa553221..06344d986eb9 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -597,7 +597,7 @@ endmenu # "Memory Debugging"
597 597
598config DEBUG_SHIRQ 598config DEBUG_SHIRQ
599 bool "Debug shared IRQ handlers" 599 bool "Debug shared IRQ handlers"
600 depends on DEBUG_KERNEL && GENERIC_HARDIRQS 600 depends on DEBUG_KERNEL
601 help 601 help
602 Enable this to generate a spurious interrupt as soon as a shared 602 Enable this to generate a spurious interrupt as soon as a shared
603 interrupt handler is registered, and just before one is deregistered. 603 interrupt handler is registered, and just before one is deregistered.
@@ -908,7 +908,7 @@ config LOCKDEP
908 bool 908 bool
909 depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT 909 depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
910 select STACKTRACE 910 select STACKTRACE
911 select FRAME_POINTER if !MIPS && !PPC && !ARM_UNWIND && !S390 && !MICROBLAZE 911 select FRAME_POINTER if !MIPS && !PPC && !ARM_UNWIND && !S390 && !MICROBLAZE && !ARC
912 select KALLSYMS 912 select KALLSYMS
913 select KALLSYMS_ALL 913 select KALLSYMS_ALL
914 914
@@ -981,6 +981,25 @@ config DEBUG_KOBJECT
981 If you say Y here, some extra kobject debugging messages will be sent 981 If you say Y here, some extra kobject debugging messages will be sent
982 to the syslog. 982 to the syslog.
983 983
984config DEBUG_KOBJECT_RELEASE
985 bool "kobject release debugging"
986 depends on DEBUG_KERNEL
987 help
988 kobjects are reference counted objects. This means that their
989 last reference count put is not predictable, and the kobject can
990 live on past the point at which a driver decides to drop it's
991 initial reference to the kobject gained on allocation. An
992 example of this would be a struct device which has just been
993 unregistered.
994
995 However, some buggy drivers assume that after such an operation,
996 the memory backing the kobject can be immediately freed. This
997 goes completely against the principles of a refcounted object.
998
999 If you say Y here, the kernel will delay the release of kobjects
1000 on the last reference count to improve the visibility of this
1001 kind of kobject release bug.
1002
984config HAVE_DEBUG_BUGVERBOSE 1003config HAVE_DEBUG_BUGVERBOSE
985 bool 1004 bool
986 1005
@@ -1347,7 +1366,7 @@ config FAULT_INJECTION_STACKTRACE_FILTER
1347 depends on FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT 1366 depends on FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT
1348 depends on !X86_64 1367 depends on !X86_64
1349 select STACKTRACE 1368 select STACKTRACE
1350 select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND 1369 select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND && !ARC
1351 help 1370 help
1352 Provide stacktrace filter for fault-injection capabilities 1371 Provide stacktrace filter for fault-injection capabilities
1353 1372
@@ -1357,7 +1376,7 @@ config LATENCYTOP
1357 depends on DEBUG_KERNEL 1376 depends on DEBUG_KERNEL
1358 depends on STACKTRACE_SUPPORT 1377 depends on STACKTRACE_SUPPORT
1359 depends on PROC_FS 1378 depends on PROC_FS
1360 select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND 1379 select FRAME_POINTER if !MIPS && !PPC && !S390 && !MICROBLAZE && !ARM_UNWIND && !ARC
1361 select KALLSYMS 1380 select KALLSYMS
1362 select KALLSYMS_ALL 1381 select KALLSYMS_ALL
1363 select STACKTRACE 1382 select STACKTRACE
@@ -1442,7 +1461,7 @@ config BACKTRACE_SELF_TEST
1442 1461
1443config RBTREE_TEST 1462config RBTREE_TEST
1444 tristate "Red-Black tree test" 1463 tristate "Red-Black tree test"
1445 depends on m && DEBUG_KERNEL 1464 depends on DEBUG_KERNEL
1446 help 1465 help
1447 A benchmark measuring the performance of the rbtree library. 1466 A benchmark measuring the performance of the rbtree library.
1448 Also includes rbtree invariant checks. 1467 Also includes rbtree invariant checks.
diff --git a/lib/Makefile b/lib/Makefile
index 7baccfd8a4e9..f3bb2cb98adf 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -13,18 +13,20 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \
13 sha1.o md5.o irq_regs.o reciprocal_div.o argv_split.o \ 13 sha1.o md5.o irq_regs.o reciprocal_div.o argv_split.o \
14 proportions.o flex_proportions.o prio_heap.o ratelimit.o show_mem.o \ 14 proportions.o flex_proportions.o prio_heap.o ratelimit.o show_mem.o \
15 is_single_threaded.o plist.o decompress.o kobject_uevent.o \ 15 is_single_threaded.o plist.o decompress.o kobject_uevent.o \
16 earlycpio.o percpu-refcount.o 16 earlycpio.o percpu-refcount.o percpu_ida.o
17 17
18obj-$(CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS) += usercopy.o 18obj-$(CONFIG_ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS) += usercopy.o
19lib-$(CONFIG_MMU) += ioremap.o 19lib-$(CONFIG_MMU) += ioremap.o
20lib-$(CONFIG_SMP) += cpumask.o 20lib-$(CONFIG_SMP) += cpumask.o
21 21
22lib-y += kobject.o klist.o 22lib-y += kobject.o klist.o
23obj-y += lockref.o
23 24
24obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \ 25obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \
25 bust_spinlocks.o hexdump.o kasprintf.o bitmap.o scatterlist.o \ 26 bust_spinlocks.o hexdump.o kasprintf.o bitmap.o scatterlist.o \
26 gcd.o lcm.o list_sort.o uuid.o flex_array.o iovec.o clz_ctz.o \ 27 gcd.o lcm.o list_sort.o uuid.o flex_array.o iovec.o clz_ctz.o \
27 bsearch.o find_last_bit.o find_next_bit.o llist.o memweight.o kfifo.o 28 bsearch.o find_last_bit.o find_next_bit.o llist.o memweight.o kfifo.o \
29 percpu_ida.o
28obj-y += string_helpers.o 30obj-y += string_helpers.o
29obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o 31obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o
30obj-y += kstrtox.o 32obj-y += kstrtox.o
diff --git a/lib/cpu_rmap.c b/lib/cpu_rmap.c
index 5fbed5caba6e..4f134d8907a7 100644
--- a/lib/cpu_rmap.c
+++ b/lib/cpu_rmap.c
@@ -8,9 +8,7 @@
8 */ 8 */
9 9
10#include <linux/cpu_rmap.h> 10#include <linux/cpu_rmap.h>
11#ifdef CONFIG_GENERIC_HARDIRQS
12#include <linux/interrupt.h> 11#include <linux/interrupt.h>
13#endif
14#include <linux/export.h> 12#include <linux/export.h>
15 13
16/* 14/*
@@ -213,8 +211,6 @@ int cpu_rmap_update(struct cpu_rmap *rmap, u16 index,
213} 211}
214EXPORT_SYMBOL(cpu_rmap_update); 212EXPORT_SYMBOL(cpu_rmap_update);
215 213
216#ifdef CONFIG_GENERIC_HARDIRQS
217
218/* Glue between IRQ affinity notifiers and CPU rmaps */ 214/* Glue between IRQ affinity notifiers and CPU rmaps */
219 215
220struct irq_glue { 216struct irq_glue {
@@ -309,5 +305,3 @@ int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq)
309 return rc; 305 return rc;
310} 306}
311EXPORT_SYMBOL(irq_cpu_rmap_add); 307EXPORT_SYMBOL(irq_cpu_rmap_add);
312
313#endif /* CONFIG_GENERIC_HARDIRQS */
diff --git a/lib/crc-t10dif.c b/lib/crc-t10dif.c
index fbbd66ed86cd..dfe6ec17c0a5 100644
--- a/lib/crc-t10dif.c
+++ b/lib/crc-t10dif.c
@@ -11,57 +11,54 @@
11#include <linux/types.h> 11#include <linux/types.h>
12#include <linux/module.h> 12#include <linux/module.h>
13#include <linux/crc-t10dif.h> 13#include <linux/crc-t10dif.h>
14#include <linux/err.h>
15#include <linux/init.h>
16#include <crypto/hash.h>
17#include <linux/static_key.h>
14 18
15/* Table generated using the following polynomium: 19static struct crypto_shash *crct10dif_tfm;
16 * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1 20static struct static_key crct10dif_fallback __read_mostly;
17 * gt: 0x8bb7
18 */
19static const __u16 t10_dif_crc_table[256] = {
20 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B,
21 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6,
22 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6,
23 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B,
24 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1,
25 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C,
26 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C,
27 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781,
28 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8,
29 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255,
30 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925,
31 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698,
32 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472,
33 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF,
34 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF,
35 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02,
36 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA,
37 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067,
38 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17,
39 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA,
40 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640,
41 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD,
42 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D,
43 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30,
44 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759,
45 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4,
46 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394,
47 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29,
48 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3,
49 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E,
50 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E,
51 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3
52};
53 21
54__u16 crc_t10dif(const unsigned char *buffer, size_t len) 22__u16 crc_t10dif(const unsigned char *buffer, size_t len)
55{ 23{
56 __u16 crc = 0; 24 struct {
57 unsigned int i; 25 struct shash_desc shash;
26 char ctx[2];
27 } desc;
28 int err;
29
30 if (static_key_false(&crct10dif_fallback))
31 return crc_t10dif_generic(0, buffer, len);
32
33 desc.shash.tfm = crct10dif_tfm;
34 desc.shash.flags = 0;
35 *(__u16 *)desc.ctx = 0;
58 36
59 for (i = 0 ; i < len ; i++) 37 err = crypto_shash_update(&desc.shash, buffer, len);
60 crc = (crc << 8) ^ t10_dif_crc_table[((crc >> 8) ^ buffer[i]) & 0xff]; 38 BUG_ON(err);
61 39
62 return crc; 40 return *(__u16 *)desc.ctx;
63} 41}
64EXPORT_SYMBOL(crc_t10dif); 42EXPORT_SYMBOL(crc_t10dif);
65 43
44static int __init crc_t10dif_mod_init(void)
45{
46 crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0);
47 if (IS_ERR(crct10dif_tfm)) {
48 static_key_slow_inc(&crct10dif_fallback);
49 crct10dif_tfm = NULL;
50 }
51 return 0;
52}
53
54static void __exit crc_t10dif_mod_fini(void)
55{
56 crypto_free_shash(crct10dif_tfm);
57}
58
59module_init(crc_t10dif_mod_init);
60module_exit(crc_t10dif_mod_fini);
61
66MODULE_DESCRIPTION("T10 DIF CRC calculation"); 62MODULE_DESCRIPTION("T10 DIF CRC calculation");
67MODULE_LICENSE("GPL"); 63MODULE_LICENSE("GPL");
64MODULE_SOFTDEP("pre: crct10dif");
diff --git a/lib/crc32.c b/lib/crc32.c
index 072fbd8234d5..410093dbe51c 100644
--- a/lib/crc32.c
+++ b/lib/crc32.c
@@ -131,11 +131,14 @@ crc32_body(u32 crc, unsigned char const *buf, size_t len, const u32 (*tab)[256])
131#endif 131#endif
132 132
133/** 133/**
134 * crc32_le() - Calculate bitwise little-endian Ethernet AUTODIN II CRC32 134 * crc32_le_generic() - Calculate bitwise little-endian Ethernet AUTODIN II
135 * @crc: seed value for computation. ~0 for Ethernet, sometimes 0 for 135 * CRC32/CRC32C
136 * other uses, or the previous crc32 value if computing incrementally. 136 * @crc: seed value for computation. ~0 for Ethernet, sometimes 0 for other
137 * @p: pointer to buffer over which CRC is run 137 * uses, or the previous crc32/crc32c value if computing incrementally.
138 * @p: pointer to buffer over which CRC32/CRC32C is run
138 * @len: length of buffer @p 139 * @len: length of buffer @p
140 * @tab: little-endian Ethernet table
141 * @polynomial: CRC32/CRC32c LE polynomial
139 */ 142 */
140static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p, 143static inline u32 __pure crc32_le_generic(u32 crc, unsigned char const *p,
141 size_t len, const u32 (*tab)[256], 144 size_t len, const u32 (*tab)[256],
@@ -201,11 +204,13 @@ EXPORT_SYMBOL(crc32_le);
201EXPORT_SYMBOL(__crc32c_le); 204EXPORT_SYMBOL(__crc32c_le);
202 205
203/** 206/**
204 * crc32_be() - Calculate bitwise big-endian Ethernet AUTODIN II CRC32 207 * crc32_be_generic() - Calculate bitwise big-endian Ethernet AUTODIN II CRC32
205 * @crc: seed value for computation. ~0 for Ethernet, sometimes 0 for 208 * @crc: seed value for computation. ~0 for Ethernet, sometimes 0 for
206 * other uses, or the previous crc32 value if computing incrementally. 209 * other uses, or the previous crc32 value if computing incrementally.
207 * @p: pointer to buffer over which CRC is run 210 * @p: pointer to buffer over which CRC32 is run
208 * @len: length of buffer @p 211 * @len: length of buffer @p
212 * @tab: big-endian Ethernet table
213 * @polynomial: CRC32 BE polynomial
209 */ 214 */
210static inline u32 __pure crc32_be_generic(u32 crc, unsigned char const *p, 215static inline u32 __pure crc32_be_generic(u32 crc, unsigned char const *p,
211 size_t len, const u32 (*tab)[256], 216 size_t len, const u32 (*tab)[256],
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 37061ede8b81..bf2c8b1043d8 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -381,19 +381,21 @@ void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
381 * debug_object_activate - debug checks when an object is activated 381 * debug_object_activate - debug checks when an object is activated
382 * @addr: address of the object 382 * @addr: address of the object
383 * @descr: pointer to an object specific debug description structure 383 * @descr: pointer to an object specific debug description structure
384 * Returns 0 for success, -EINVAL for check failed.
384 */ 385 */
385void debug_object_activate(void *addr, struct debug_obj_descr *descr) 386int debug_object_activate(void *addr, struct debug_obj_descr *descr)
386{ 387{
387 enum debug_obj_state state; 388 enum debug_obj_state state;
388 struct debug_bucket *db; 389 struct debug_bucket *db;
389 struct debug_obj *obj; 390 struct debug_obj *obj;
390 unsigned long flags; 391 unsigned long flags;
392 int ret;
391 struct debug_obj o = { .object = addr, 393 struct debug_obj o = { .object = addr,
392 .state = ODEBUG_STATE_NOTAVAILABLE, 394 .state = ODEBUG_STATE_NOTAVAILABLE,
393 .descr = descr }; 395 .descr = descr };
394 396
395 if (!debug_objects_enabled) 397 if (!debug_objects_enabled)
396 return; 398 return 0;
397 399
398 db = get_bucket((unsigned long) addr); 400 db = get_bucket((unsigned long) addr);
399 401
@@ -405,23 +407,26 @@ void debug_object_activate(void *addr, struct debug_obj_descr *descr)
405 case ODEBUG_STATE_INIT: 407 case ODEBUG_STATE_INIT:
406 case ODEBUG_STATE_INACTIVE: 408 case ODEBUG_STATE_INACTIVE:
407 obj->state = ODEBUG_STATE_ACTIVE; 409 obj->state = ODEBUG_STATE_ACTIVE;
410 ret = 0;
408 break; 411 break;
409 412
410 case ODEBUG_STATE_ACTIVE: 413 case ODEBUG_STATE_ACTIVE:
411 debug_print_object(obj, "activate"); 414 debug_print_object(obj, "activate");
412 state = obj->state; 415 state = obj->state;
413 raw_spin_unlock_irqrestore(&db->lock, flags); 416 raw_spin_unlock_irqrestore(&db->lock, flags);
414 debug_object_fixup(descr->fixup_activate, addr, state); 417 ret = debug_object_fixup(descr->fixup_activate, addr, state);
415 return; 418 return ret ? -EINVAL : 0;
416 419
417 case ODEBUG_STATE_DESTROYED: 420 case ODEBUG_STATE_DESTROYED:
418 debug_print_object(obj, "activate"); 421 debug_print_object(obj, "activate");
422 ret = -EINVAL;
419 break; 423 break;
420 default: 424 default:
425 ret = 0;
421 break; 426 break;
422 } 427 }
423 raw_spin_unlock_irqrestore(&db->lock, flags); 428 raw_spin_unlock_irqrestore(&db->lock, flags);
424 return; 429 return ret;
425 } 430 }
426 431
427 raw_spin_unlock_irqrestore(&db->lock, flags); 432 raw_spin_unlock_irqrestore(&db->lock, flags);
@@ -431,8 +436,11 @@ void debug_object_activate(void *addr, struct debug_obj_descr *descr)
431 * true or not. 436 * true or not.
432 */ 437 */
433 if (debug_object_fixup(descr->fixup_activate, addr, 438 if (debug_object_fixup(descr->fixup_activate, addr,
434 ODEBUG_STATE_NOTAVAILABLE)) 439 ODEBUG_STATE_NOTAVAILABLE)) {
435 debug_print_object(&o, "activate"); 440 debug_print_object(&o, "activate");
441 return -EINVAL;
442 }
443 return 0;
436} 444}
437 445
438/** 446/**
diff --git a/lib/decompress_inflate.c b/lib/decompress_inflate.c
index 19ff89e34eec..d619b28c456f 100644
--- a/lib/decompress_inflate.c
+++ b/lib/decompress_inflate.c
@@ -48,7 +48,7 @@ STATIC int INIT gunzip(unsigned char *buf, int len,
48 out_len = 0x8000; /* 32 K */ 48 out_len = 0x8000; /* 32 K */
49 out_buf = malloc(out_len); 49 out_buf = malloc(out_len);
50 } else { 50 } else {
51 out_len = 0x7fffffff; /* no limit */ 51 out_len = ((size_t)~0) - (size_t)out_buf; /* no limit */
52 } 52 }
53 if (!out_buf) { 53 if (!out_buf) {
54 error("Out of memory while allocating output buffer"); 54 error("Out of memory while allocating output buffer");
diff --git a/lib/div64.c b/lib/div64.c
index a163b6caef73..4382ad77777e 100644
--- a/lib/div64.c
+++ b/lib/div64.c
@@ -79,6 +79,46 @@ EXPORT_SYMBOL(div_s64_rem);
79#endif 79#endif
80 80
81/** 81/**
82 * div64_u64_rem - unsigned 64bit divide with 64bit divisor and remainder
83 * @dividend: 64bit dividend
84 * @divisor: 64bit divisor
85 * @remainder: 64bit remainder
86 *
87 * This implementation is a comparable to algorithm used by div64_u64.
88 * But this operation, which includes math for calculating the remainder,
89 * is kept distinct to avoid slowing down the div64_u64 operation on 32bit
90 * systems.
91 */
92#ifndef div64_u64_rem
93u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
94{
95 u32 high = divisor >> 32;
96 u64 quot;
97
98 if (high == 0) {
99 u32 rem32;
100 quot = div_u64_rem(dividend, divisor, &rem32);
101 *remainder = rem32;
102 } else {
103 int n = 1 + fls(high);
104 quot = div_u64(dividend >> n, divisor >> n);
105
106 if (quot != 0)
107 quot--;
108
109 *remainder = dividend - quot * divisor;
110 if (*remainder >= divisor) {
111 quot++;
112 *remainder -= divisor;
113 }
114 }
115
116 return quot;
117}
118EXPORT_SYMBOL(div64_u64_rem);
119#endif
120
121/**
82 * div64_u64 - unsigned 64bit divide with 64bit divisor 122 * div64_u64 - unsigned 64bit divide with 64bit divisor
83 * @dividend: 64bit dividend 123 * @dividend: 64bit dividend
84 * @divisor: 64bit divisor 124 * @divisor: 64bit divisor
diff --git a/lib/dump_stack.c b/lib/dump_stack.c
index c03154173cc7..f23b63f0a1c3 100644
--- a/lib/dump_stack.c
+++ b/lib/dump_stack.c
@@ -23,7 +23,7 @@ static void __dump_stack(void)
23#ifdef CONFIG_SMP 23#ifdef CONFIG_SMP
24static atomic_t dump_lock = ATOMIC_INIT(-1); 24static atomic_t dump_lock = ATOMIC_INIT(-1);
25 25
26void dump_stack(void) 26asmlinkage void dump_stack(void)
27{ 27{
28 int was_locked; 28 int was_locked;
29 int old; 29 int old;
@@ -55,7 +55,7 @@ retry:
55 preempt_enable(); 55 preempt_enable();
56} 56}
57#else 57#else
58void dump_stack(void) 58asmlinkage void dump_stack(void)
59{ 59{
60 __dump_stack(); 60 __dump_stack();
61} 61}
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index 99fec3ae405a..c37aeacd7651 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -309,7 +309,7 @@ static int ddebug_parse_query(char *words[], int nwords,
309 struct ddebug_query *query, const char *modname) 309 struct ddebug_query *query, const char *modname)
310{ 310{
311 unsigned int i; 311 unsigned int i;
312 int rc; 312 int rc = 0;
313 313
314 /* check we have an even number of words */ 314 /* check we have an even number of words */
315 if (nwords % 2 != 0) { 315 if (nwords % 2 != 0) {
diff --git a/lib/earlycpio.c b/lib/earlycpio.c
index 7aa7ce250c94..3eb3e4722b8e 100644
--- a/lib/earlycpio.c
+++ b/lib/earlycpio.c
@@ -49,22 +49,23 @@ enum cpio_fields {
49 49
50/** 50/**
51 * cpio_data find_cpio_data - Search for files in an uncompressed cpio 51 * cpio_data find_cpio_data - Search for files in an uncompressed cpio
52 * @path: The directory to search for, including a slash at the end 52 * @path: The directory to search for, including a slash at the end
53 * @data: Pointer to the the cpio archive or a header inside 53 * @data: Pointer to the the cpio archive or a header inside
54 * @len: Remaining length of the cpio based on data pointer 54 * @len: Remaining length of the cpio based on data pointer
55 * @offset: When a matching file is found, this is the offset to the 55 * @nextoff: When a matching file is found, this is the offset from the
56 * beginning of the cpio. It can be used to iterate through 56 * beginning of the cpio to the beginning of the next file, not the
57 * the cpio to find all files inside of a directory path 57 * matching file itself. It can be used to iterate through the cpio
58 * to find all files inside of a directory path.
58 * 59 *
59 * @return: struct cpio_data containing the address, length and 60 * @return: struct cpio_data containing the address, length and
60 * filename (with the directory path cut off) of the found file. 61 * filename (with the directory path cut off) of the found file.
61 * If you search for a filename and not for files in a directory, 62 * If you search for a filename and not for files in a directory,
62 * pass the absolute path of the filename in the cpio and make sure 63 * pass the absolute path of the filename in the cpio and make sure
63 * the match returned an empty filename string. 64 * the match returned an empty filename string.
64 */ 65 */
65 66
66struct cpio_data find_cpio_data(const char *path, void *data, 67struct cpio_data find_cpio_data(const char *path, void *data,
67 size_t len, long *offset) 68 size_t len, long *nextoff)
68{ 69{
69 const size_t cpio_header_len = 8*C_NFIELDS - 2; 70 const size_t cpio_header_len = 8*C_NFIELDS - 2;
70 struct cpio_data cd = { NULL, 0, "" }; 71 struct cpio_data cd = { NULL, 0, "" };
@@ -124,7 +125,7 @@ struct cpio_data find_cpio_data(const char *path, void *data,
124 if ((ch[C_MODE] & 0170000) == 0100000 && 125 if ((ch[C_MODE] & 0170000) == 0100000 &&
125 ch[C_NAMESIZE] >= mypathsize && 126 ch[C_NAMESIZE] >= mypathsize &&
126 !memcmp(p, path, mypathsize)) { 127 !memcmp(p, path, mypathsize)) {
127 *offset = (long)nptr - (long)data; 128 *nextoff = (long)nptr - (long)data;
128 if (ch[C_NAMESIZE] - mypathsize >= MAX_CPIO_FILE_NAME) { 129 if (ch[C_NAMESIZE] - mypathsize >= MAX_CPIO_FILE_NAME) {
129 pr_warn( 130 pr_warn(
130 "File %s exceeding MAX_CPIO_FILE_NAME [%d]\n", 131 "File %s exceeding MAX_CPIO_FILE_NAME [%d]\n",
diff --git a/lib/genalloc.c b/lib/genalloc.c
index b35cfa9bc3d4..26cf20be72b7 100644
--- a/lib/genalloc.c
+++ b/lib/genalloc.c
@@ -37,6 +37,11 @@
37#include <linux/of_address.h> 37#include <linux/of_address.h>
38#include <linux/of_device.h> 38#include <linux/of_device.h>
39 39
40static inline size_t chunk_size(const struct gen_pool_chunk *chunk)
41{
42 return chunk->end_addr - chunk->start_addr + 1;
43}
44
40static int set_bits_ll(unsigned long *addr, unsigned long mask_to_set) 45static int set_bits_ll(unsigned long *addr, unsigned long mask_to_set)
41{ 46{
42 unsigned long val, nval; 47 unsigned long val, nval;
@@ -182,13 +187,13 @@ int gen_pool_add_virt(struct gen_pool *pool, unsigned long virt, phys_addr_t phy
182 int nbytes = sizeof(struct gen_pool_chunk) + 187 int nbytes = sizeof(struct gen_pool_chunk) +
183 BITS_TO_LONGS(nbits) * sizeof(long); 188 BITS_TO_LONGS(nbits) * sizeof(long);
184 189
185 chunk = kmalloc_node(nbytes, GFP_KERNEL | __GFP_ZERO, nid); 190 chunk = kzalloc_node(nbytes, GFP_KERNEL, nid);
186 if (unlikely(chunk == NULL)) 191 if (unlikely(chunk == NULL))
187 return -ENOMEM; 192 return -ENOMEM;
188 193
189 chunk->phys_addr = phys; 194 chunk->phys_addr = phys;
190 chunk->start_addr = virt; 195 chunk->start_addr = virt;
191 chunk->end_addr = virt + size; 196 chunk->end_addr = virt + size - 1;
192 atomic_set(&chunk->avail, size); 197 atomic_set(&chunk->avail, size);
193 198
194 spin_lock(&pool->lock); 199 spin_lock(&pool->lock);
@@ -213,7 +218,7 @@ phys_addr_t gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long addr)
213 218
214 rcu_read_lock(); 219 rcu_read_lock();
215 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) { 220 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
216 if (addr >= chunk->start_addr && addr < chunk->end_addr) { 221 if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
217 paddr = chunk->phys_addr + (addr - chunk->start_addr); 222 paddr = chunk->phys_addr + (addr - chunk->start_addr);
218 break; 223 break;
219 } 224 }
@@ -242,7 +247,7 @@ void gen_pool_destroy(struct gen_pool *pool)
242 chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk); 247 chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
243 list_del(&chunk->next_chunk); 248 list_del(&chunk->next_chunk);
244 249
245 end_bit = (chunk->end_addr - chunk->start_addr) >> order; 250 end_bit = chunk_size(chunk) >> order;
246 bit = find_next_bit(chunk->bits, end_bit, 0); 251 bit = find_next_bit(chunk->bits, end_bit, 0);
247 BUG_ON(bit < end_bit); 252 BUG_ON(bit < end_bit);
248 253
@@ -283,7 +288,7 @@ unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
283 if (size > atomic_read(&chunk->avail)) 288 if (size > atomic_read(&chunk->avail))
284 continue; 289 continue;
285 290
286 end_bit = (chunk->end_addr - chunk->start_addr) >> order; 291 end_bit = chunk_size(chunk) >> order;
287retry: 292retry:
288 start_bit = pool->algo(chunk->bits, end_bit, start_bit, nbits, 293 start_bit = pool->algo(chunk->bits, end_bit, start_bit, nbits,
289 pool->data); 294 pool->data);
@@ -330,8 +335,8 @@ void gen_pool_free(struct gen_pool *pool, unsigned long addr, size_t size)
330 nbits = (size + (1UL << order) - 1) >> order; 335 nbits = (size + (1UL << order) - 1) >> order;
331 rcu_read_lock(); 336 rcu_read_lock();
332 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) { 337 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
333 if (addr >= chunk->start_addr && addr < chunk->end_addr) { 338 if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
334 BUG_ON(addr + size > chunk->end_addr); 339 BUG_ON(addr + size - 1 > chunk->end_addr);
335 start_bit = (addr - chunk->start_addr) >> order; 340 start_bit = (addr - chunk->start_addr) >> order;
336 remain = bitmap_clear_ll(chunk->bits, start_bit, nbits); 341 remain = bitmap_clear_ll(chunk->bits, start_bit, nbits);
337 BUG_ON(remain); 342 BUG_ON(remain);
@@ -400,7 +405,7 @@ size_t gen_pool_size(struct gen_pool *pool)
400 405
401 rcu_read_lock(); 406 rcu_read_lock();
402 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) 407 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk)
403 size += chunk->end_addr - chunk->start_addr; 408 size += chunk_size(chunk);
404 rcu_read_unlock(); 409 rcu_read_unlock();
405 return size; 410 return size;
406} 411}
@@ -519,7 +524,6 @@ struct gen_pool *devm_gen_pool_create(struct device *dev, int min_alloc_order,
519/** 524/**
520 * dev_get_gen_pool - Obtain the gen_pool (if any) for a device 525 * dev_get_gen_pool - Obtain the gen_pool (if any) for a device
521 * @dev: device to retrieve the gen_pool from 526 * @dev: device to retrieve the gen_pool from
522 * @name: Optional name for the gen_pool, usually NULL
523 * 527 *
524 * Returns the gen_pool for the device if one is present, or NULL. 528 * Returns the gen_pool for the device if one is present, or NULL.
525 */ 529 */
diff --git a/lib/hexdump.c b/lib/hexdump.c
index 3f0494c9d57a..8499c810909a 100644
--- a/lib/hexdump.c
+++ b/lib/hexdump.c
@@ -14,6 +14,8 @@
14 14
15const char hex_asc[] = "0123456789abcdef"; 15const char hex_asc[] = "0123456789abcdef";
16EXPORT_SYMBOL(hex_asc); 16EXPORT_SYMBOL(hex_asc);
17const char hex_asc_upper[] = "0123456789ABCDEF";
18EXPORT_SYMBOL(hex_asc_upper);
17 19
18/** 20/**
19 * hex_to_bin - convert a hex digit to its real value 21 * hex_to_bin - convert a hex digit to its real value
diff --git a/lib/kobject.c b/lib/kobject.c
index 4a1f33d43548..669bf190d4fb 100644
--- a/lib/kobject.c
+++ b/lib/kobject.c
@@ -545,8 +545,8 @@ static void kobject_cleanup(struct kobject *kobj)
545 struct kobj_type *t = get_ktype(kobj); 545 struct kobj_type *t = get_ktype(kobj);
546 const char *name = kobj->name; 546 const char *name = kobj->name;
547 547
548 pr_debug("kobject: '%s' (%p): %s\n", 548 pr_debug("kobject: '%s' (%p): %s, parent %p\n",
549 kobject_name(kobj), kobj, __func__); 549 kobject_name(kobj), kobj, __func__, kobj->parent);
550 550
551 if (t && !t->release) 551 if (t && !t->release)
552 pr_debug("kobject: '%s' (%p): does not have a release() " 552 pr_debug("kobject: '%s' (%p): does not have a release() "
@@ -580,9 +580,25 @@ static void kobject_cleanup(struct kobject *kobj)
580 } 580 }
581} 581}
582 582
583#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
584static void kobject_delayed_cleanup(struct work_struct *work)
585{
586 kobject_cleanup(container_of(to_delayed_work(work),
587 struct kobject, release));
588}
589#endif
590
583static void kobject_release(struct kref *kref) 591static void kobject_release(struct kref *kref)
584{ 592{
585 kobject_cleanup(container_of(kref, struct kobject, kref)); 593 struct kobject *kobj = container_of(kref, struct kobject, kref);
594#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
595 pr_debug("kobject: '%s' (%p): %s, parent %p (delayed)\n",
596 kobject_name(kobj), kobj, __func__, kobj->parent);
597 INIT_DELAYED_WORK(&kobj->release, kobject_delayed_cleanup);
598 schedule_delayed_work(&kobj->release, HZ);
599#else
600 kobject_cleanup(kobj);
601#endif
586} 602}
587 603
588/** 604/**
@@ -915,6 +931,18 @@ const struct kobj_ns_type_operations *kobj_ns_ops(struct kobject *kobj)
915 return kobj_child_ns_ops(kobj->parent); 931 return kobj_child_ns_ops(kobj->parent);
916} 932}
917 933
934bool kobj_ns_current_may_mount(enum kobj_ns_type type)
935{
936 bool may_mount = true;
937
938 spin_lock(&kobj_ns_type_lock);
939 if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) &&
940 kobj_ns_ops_tbl[type])
941 may_mount = kobj_ns_ops_tbl[type]->current_may_mount();
942 spin_unlock(&kobj_ns_type_lock);
943
944 return may_mount;
945}
918 946
919void *kobj_ns_grab_current(enum kobj_ns_type type) 947void *kobj_ns_grab_current(enum kobj_ns_type type)
920{ 948{
diff --git a/lib/lockref.c b/lib/lockref.c
new file mode 100644
index 000000000000..6f9d434c1521
--- /dev/null
+++ b/lib/lockref.c
@@ -0,0 +1,183 @@
1#include <linux/export.h>
2#include <linux/lockref.h>
3
4#ifdef CONFIG_CMPXCHG_LOCKREF
5
6/*
7 * Allow weakly-ordered memory architectures to provide barrier-less
8 * cmpxchg semantics for lockref updates.
9 */
10#ifndef cmpxchg64_relaxed
11# define cmpxchg64_relaxed cmpxchg64
12#endif
13
14/*
15 * Allow architectures to override the default cpu_relax() within CMPXCHG_LOOP.
16 * This is useful for architectures with an expensive cpu_relax().
17 */
18#ifndef arch_mutex_cpu_relax
19# define arch_mutex_cpu_relax() cpu_relax()
20#endif
21
22/*
23 * Note that the "cmpxchg()" reloads the "old" value for the
24 * failure case.
25 */
26#define CMPXCHG_LOOP(CODE, SUCCESS) do { \
27 struct lockref old; \
28 BUILD_BUG_ON(sizeof(old) != 8); \
29 old.lock_count = ACCESS_ONCE(lockref->lock_count); \
30 while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \
31 struct lockref new = old, prev = old; \
32 CODE \
33 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
34 old.lock_count, \
35 new.lock_count); \
36 if (likely(old.lock_count == prev.lock_count)) { \
37 SUCCESS; \
38 } \
39 arch_mutex_cpu_relax(); \
40 } \
41} while (0)
42
43#else
44
45#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
46
47#endif
48
49/**
50 * lockref_get - Increments reference count unconditionally
51 * @lockref: pointer to lockref structure
52 *
53 * This operation is only valid if you already hold a reference
54 * to the object, so you know the count cannot be zero.
55 */
56void lockref_get(struct lockref *lockref)
57{
58 CMPXCHG_LOOP(
59 new.count++;
60 ,
61 return;
62 );
63
64 spin_lock(&lockref->lock);
65 lockref->count++;
66 spin_unlock(&lockref->lock);
67}
68EXPORT_SYMBOL(lockref_get);
69
70/**
71 * lockref_get_not_zero - Increments count unless the count is 0
72 * @lockref: pointer to lockref structure
73 * Return: 1 if count updated successfully or 0 if count was zero
74 */
75int lockref_get_not_zero(struct lockref *lockref)
76{
77 int retval;
78
79 CMPXCHG_LOOP(
80 new.count++;
81 if (!old.count)
82 return 0;
83 ,
84 return 1;
85 );
86
87 spin_lock(&lockref->lock);
88 retval = 0;
89 if (lockref->count) {
90 lockref->count++;
91 retval = 1;
92 }
93 spin_unlock(&lockref->lock);
94 return retval;
95}
96EXPORT_SYMBOL(lockref_get_not_zero);
97
98/**
99 * lockref_get_or_lock - Increments count unless the count is 0
100 * @lockref: pointer to lockref structure
101 * Return: 1 if count updated successfully or 0 if count was zero
102 * and we got the lock instead.
103 */
104int lockref_get_or_lock(struct lockref *lockref)
105{
106 CMPXCHG_LOOP(
107 new.count++;
108 if (!old.count)
109 break;
110 ,
111 return 1;
112 );
113
114 spin_lock(&lockref->lock);
115 if (!lockref->count)
116 return 0;
117 lockref->count++;
118 spin_unlock(&lockref->lock);
119 return 1;
120}
121EXPORT_SYMBOL(lockref_get_or_lock);
122
123/**
124 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
125 * @lockref: pointer to lockref structure
126 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
127 */
128int lockref_put_or_lock(struct lockref *lockref)
129{
130 CMPXCHG_LOOP(
131 new.count--;
132 if (old.count <= 1)
133 break;
134 ,
135 return 1;
136 );
137
138 spin_lock(&lockref->lock);
139 if (lockref->count <= 1)
140 return 0;
141 lockref->count--;
142 spin_unlock(&lockref->lock);
143 return 1;
144}
145EXPORT_SYMBOL(lockref_put_or_lock);
146
147/**
148 * lockref_mark_dead - mark lockref dead
149 * @lockref: pointer to lockref structure
150 */
151void lockref_mark_dead(struct lockref *lockref)
152{
153 assert_spin_locked(&lockref->lock);
154 lockref->count = -128;
155}
156
157/**
158 * lockref_get_not_dead - Increments count unless the ref is dead
159 * @lockref: pointer to lockref structure
160 * Return: 1 if count updated successfully or 0 if lockref was dead
161 */
162int lockref_get_not_dead(struct lockref *lockref)
163{
164 int retval;
165
166 CMPXCHG_LOOP(
167 new.count++;
168 if ((int)old.count < 0)
169 return 0;
170 ,
171 return 1;
172 );
173
174 spin_lock(&lockref->lock);
175 retval = 0;
176 if ((int) lockref->count >= 0) {
177 lockref->count++;
178 retval = 1;
179 }
180 spin_unlock(&lockref->lock);
181 return retval;
182}
183EXPORT_SYMBOL(lockref_get_not_dead);
diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c
index 411be80ddb46..df6839e3ce08 100644
--- a/lib/lz4/lz4_decompress.c
+++ b/lib/lz4/lz4_decompress.c
@@ -283,8 +283,8 @@ _output_error:
283 return (int) (-(((char *) ip) - source)); 283 return (int) (-(((char *) ip) - source));
284} 284}
285 285
286int lz4_decompress(const char *src, size_t *src_len, char *dest, 286int lz4_decompress(const unsigned char *src, size_t *src_len,
287 size_t actual_dest_len) 287 unsigned char *dest, size_t actual_dest_len)
288{ 288{
289 int ret = -1; 289 int ret = -1;
290 int input_len = 0; 290 int input_len = 0;
@@ -302,8 +302,8 @@ exit_0:
302EXPORT_SYMBOL(lz4_decompress); 302EXPORT_SYMBOL(lz4_decompress);
303#endif 303#endif
304 304
305int lz4_decompress_unknownoutputsize(const char *src, size_t src_len, 305int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
306 char *dest, size_t *dest_len) 306 unsigned char *dest, size_t *dest_len)
307{ 307{
308 int ret = -1; 308 int ret = -1;
309 int out_len = 0; 309 int out_len = 0;
diff --git a/lib/percpu_ida.c b/lib/percpu_ida.c
new file mode 100644
index 000000000000..bab1ba2a4c71
--- /dev/null
+++ b/lib/percpu_ida.c
@@ -0,0 +1,335 @@
1/*
2 * Percpu IDA library
3 *
4 * Copyright (C) 2013 Datera, Inc. Kent Overstreet
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation; either version 2, or (at
9 * your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
15 */
16
17#include <linux/bitmap.h>
18#include <linux/bitops.h>
19#include <linux/bug.h>
20#include <linux/err.h>
21#include <linux/export.h>
22#include <linux/hardirq.h>
23#include <linux/idr.h>
24#include <linux/init.h>
25#include <linux/kernel.h>
26#include <linux/percpu.h>
27#include <linux/sched.h>
28#include <linux/slab.h>
29#include <linux/string.h>
30#include <linux/spinlock.h>
31#include <linux/percpu_ida.h>
32
33/*
34 * Number of tags we move between the percpu freelist and the global freelist at
35 * a time
36 */
37#define IDA_PCPU_BATCH_MOVE 32U
38
39/* Max size of percpu freelist, */
40#define IDA_PCPU_SIZE ((IDA_PCPU_BATCH_MOVE * 3) / 2)
41
42struct percpu_ida_cpu {
43 /*
44 * Even though this is percpu, we need a lock for tag stealing by remote
45 * CPUs:
46 */
47 spinlock_t lock;
48
49 /* nr_free/freelist form a stack of free IDs */
50 unsigned nr_free;
51 unsigned freelist[];
52};
53
54static inline void move_tags(unsigned *dst, unsigned *dst_nr,
55 unsigned *src, unsigned *src_nr,
56 unsigned nr)
57{
58 *src_nr -= nr;
59 memcpy(dst + *dst_nr, src + *src_nr, sizeof(unsigned) * nr);
60 *dst_nr += nr;
61}
62
63/*
64 * Try to steal tags from a remote cpu's percpu freelist.
65 *
66 * We first check how many percpu freelists have tags - we don't steal tags
67 * unless enough percpu freelists have tags on them that it's possible more than
68 * half the total tags could be stuck on remote percpu freelists.
69 *
70 * Then we iterate through the cpus until we find some tags - we don't attempt
71 * to find the "best" cpu to steal from, to keep cacheline bouncing to a
72 * minimum.
73 */
74static inline void steal_tags(struct percpu_ida *pool,
75 struct percpu_ida_cpu *tags)
76{
77 unsigned cpus_have_tags, cpu = pool->cpu_last_stolen;
78 struct percpu_ida_cpu *remote;
79
80 for (cpus_have_tags = cpumask_weight(&pool->cpus_have_tags);
81 cpus_have_tags * IDA_PCPU_SIZE > pool->nr_tags / 2;
82 cpus_have_tags--) {
83 cpu = cpumask_next(cpu, &pool->cpus_have_tags);
84
85 if (cpu >= nr_cpu_ids) {
86 cpu = cpumask_first(&pool->cpus_have_tags);
87 if (cpu >= nr_cpu_ids)
88 BUG();
89 }
90
91 pool->cpu_last_stolen = cpu;
92 remote = per_cpu_ptr(pool->tag_cpu, cpu);
93
94 cpumask_clear_cpu(cpu, &pool->cpus_have_tags);
95
96 if (remote == tags)
97 continue;
98
99 spin_lock(&remote->lock);
100
101 if (remote->nr_free) {
102 memcpy(tags->freelist,
103 remote->freelist,
104 sizeof(unsigned) * remote->nr_free);
105
106 tags->nr_free = remote->nr_free;
107 remote->nr_free = 0;
108 }
109
110 spin_unlock(&remote->lock);
111
112 if (tags->nr_free)
113 break;
114 }
115}
116
117/*
118 * Pop up to IDA_PCPU_BATCH_MOVE IDs off the global freelist, and push them onto
119 * our percpu freelist:
120 */
121static inline void alloc_global_tags(struct percpu_ida *pool,
122 struct percpu_ida_cpu *tags)
123{
124 move_tags(tags->freelist, &tags->nr_free,
125 pool->freelist, &pool->nr_free,
126 min(pool->nr_free, IDA_PCPU_BATCH_MOVE));
127}
128
129static inline unsigned alloc_local_tag(struct percpu_ida *pool,
130 struct percpu_ida_cpu *tags)
131{
132 int tag = -ENOSPC;
133
134 spin_lock(&tags->lock);
135 if (tags->nr_free)
136 tag = tags->freelist[--tags->nr_free];
137 spin_unlock(&tags->lock);
138
139 return tag;
140}
141
142/**
143 * percpu_ida_alloc - allocate a tag
144 * @pool: pool to allocate from
145 * @gfp: gfp flags
146 *
147 * Returns a tag - an integer in the range [0..nr_tags) (passed to
148 * tag_pool_init()), or otherwise -ENOSPC on allocation failure.
149 *
150 * Safe to be called from interrupt context (assuming it isn't passed
151 * __GFP_WAIT, of course).
152 *
153 * @gfp indicates whether or not to wait until a free id is available (it's not
154 * used for internal memory allocations); thus if passed __GFP_WAIT we may sleep
155 * however long it takes until another thread frees an id (same semantics as a
156 * mempool).
157 *
158 * Will not fail if passed __GFP_WAIT.
159 */
160int percpu_ida_alloc(struct percpu_ida *pool, gfp_t gfp)
161{
162 DEFINE_WAIT(wait);
163 struct percpu_ida_cpu *tags;
164 unsigned long flags;
165 int tag;
166
167 local_irq_save(flags);
168 tags = this_cpu_ptr(pool->tag_cpu);
169
170 /* Fastpath */
171 tag = alloc_local_tag(pool, tags);
172 if (likely(tag >= 0)) {
173 local_irq_restore(flags);
174 return tag;
175 }
176
177 while (1) {
178 spin_lock(&pool->lock);
179
180 /*
181 * prepare_to_wait() must come before steal_tags(), in case
182 * percpu_ida_free() on another cpu flips a bit in
183 * cpus_have_tags
184 *
185 * global lock held and irqs disabled, don't need percpu lock
186 */
187 prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
188
189 if (!tags->nr_free)
190 alloc_global_tags(pool, tags);
191 if (!tags->nr_free)
192 steal_tags(pool, tags);
193
194 if (tags->nr_free) {
195 tag = tags->freelist[--tags->nr_free];
196 if (tags->nr_free)
197 cpumask_set_cpu(smp_processor_id(),
198 &pool->cpus_have_tags);
199 }
200
201 spin_unlock(&pool->lock);
202 local_irq_restore(flags);
203
204 if (tag >= 0 || !(gfp & __GFP_WAIT))
205 break;
206
207 schedule();
208
209 local_irq_save(flags);
210 tags = this_cpu_ptr(pool->tag_cpu);
211 }
212
213 finish_wait(&pool->wait, &wait);
214 return tag;
215}
216EXPORT_SYMBOL_GPL(percpu_ida_alloc);
217
218/**
219 * percpu_ida_free - free a tag
220 * @pool: pool @tag was allocated from
221 * @tag: a tag previously allocated with percpu_ida_alloc()
222 *
223 * Safe to be called from interrupt context.
224 */
225void percpu_ida_free(struct percpu_ida *pool, unsigned tag)
226{
227 struct percpu_ida_cpu *tags;
228 unsigned long flags;
229 unsigned nr_free;
230
231 BUG_ON(tag >= pool->nr_tags);
232
233 local_irq_save(flags);
234 tags = this_cpu_ptr(pool->tag_cpu);
235
236 spin_lock(&tags->lock);
237 tags->freelist[tags->nr_free++] = tag;
238
239 nr_free = tags->nr_free;
240 spin_unlock(&tags->lock);
241
242 if (nr_free == 1) {
243 cpumask_set_cpu(smp_processor_id(),
244 &pool->cpus_have_tags);
245 wake_up(&pool->wait);
246 }
247
248 if (nr_free == IDA_PCPU_SIZE) {
249 spin_lock(&pool->lock);
250
251 /*
252 * Global lock held and irqs disabled, don't need percpu
253 * lock
254 */
255 if (tags->nr_free == IDA_PCPU_SIZE) {
256 move_tags(pool->freelist, &pool->nr_free,
257 tags->freelist, &tags->nr_free,
258 IDA_PCPU_BATCH_MOVE);
259
260 wake_up(&pool->wait);
261 }
262 spin_unlock(&pool->lock);
263 }
264
265 local_irq_restore(flags);
266}
267EXPORT_SYMBOL_GPL(percpu_ida_free);
268
269/**
270 * percpu_ida_destroy - release a tag pool's resources
271 * @pool: pool to free
272 *
273 * Frees the resources allocated by percpu_ida_init().
274 */
275void percpu_ida_destroy(struct percpu_ida *pool)
276{
277 free_percpu(pool->tag_cpu);
278 free_pages((unsigned long) pool->freelist,
279 get_order(pool->nr_tags * sizeof(unsigned)));
280}
281EXPORT_SYMBOL_GPL(percpu_ida_destroy);
282
283/**
284 * percpu_ida_init - initialize a percpu tag pool
285 * @pool: pool to initialize
286 * @nr_tags: number of tags that will be available for allocation
287 *
288 * Initializes @pool so that it can be used to allocate tags - integers in the
289 * range [0, nr_tags). Typically, they'll be used by driver code to refer to a
290 * preallocated array of tag structures.
291 *
292 * Allocation is percpu, but sharding is limited by nr_tags - for best
293 * performance, the workload should not span more cpus than nr_tags / 128.
294 */
295int percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags)
296{
297 unsigned i, cpu, order;
298
299 memset(pool, 0, sizeof(*pool));
300
301 init_waitqueue_head(&pool->wait);
302 spin_lock_init(&pool->lock);
303 pool->nr_tags = nr_tags;
304
305 /* Guard against overflow */
306 if (nr_tags > (unsigned) INT_MAX + 1) {
307 pr_err("percpu_ida_init(): nr_tags too large\n");
308 return -EINVAL;
309 }
310
311 order = get_order(nr_tags * sizeof(unsigned));
312 pool->freelist = (void *) __get_free_pages(GFP_KERNEL, order);
313 if (!pool->freelist)
314 return -ENOMEM;
315
316 for (i = 0; i < nr_tags; i++)
317 pool->freelist[i] = i;
318
319 pool->nr_free = nr_tags;
320
321 pool->tag_cpu = __alloc_percpu(sizeof(struct percpu_ida_cpu) +
322 IDA_PCPU_SIZE * sizeof(unsigned),
323 sizeof(unsigned));
324 if (!pool->tag_cpu)
325 goto err;
326
327 for_each_possible_cpu(cpu)
328 spin_lock_init(&per_cpu_ptr(pool->tag_cpu, cpu)->lock);
329
330 return 0;
331err:
332 percpu_ida_destroy(pool);
333 return -ENOMEM;
334}
335EXPORT_SYMBOL_GPL(percpu_ida_init);
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index e7964296fd50..7811ed3b4e70 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -32,6 +32,7 @@
32#include <linux/string.h> 32#include <linux/string.h>
33#include <linux/bitops.h> 33#include <linux/bitops.h>
34#include <linux/rcupdate.h> 34#include <linux/rcupdate.h>
35#include <linux/hardirq.h> /* in_interrupt() */
35 36
36 37
37#ifdef __KERNEL__ 38#ifdef __KERNEL__
@@ -207,7 +208,12 @@ radix_tree_node_alloc(struct radix_tree_root *root)
207 struct radix_tree_node *ret = NULL; 208 struct radix_tree_node *ret = NULL;
208 gfp_t gfp_mask = root_gfp_mask(root); 209 gfp_t gfp_mask = root_gfp_mask(root);
209 210
210 if (!(gfp_mask & __GFP_WAIT)) { 211 /*
212 * Preload code isn't irq safe and it doesn't make sence to use
213 * preloading in the interrupt anyway as all the allocations have to
214 * be atomic. So just do normal allocation when in interrupt.
215 */
216 if (!(gfp_mask & __GFP_WAIT) && !in_interrupt()) {
211 struct radix_tree_preload *rtp; 217 struct radix_tree_preload *rtp;
212 218
213 /* 219 /*
@@ -264,7 +270,7 @@ radix_tree_node_free(struct radix_tree_node *node)
264 * To make use of this facility, the radix tree must be initialised without 270 * To make use of this facility, the radix tree must be initialised without
265 * __GFP_WAIT being passed to INIT_RADIX_TREE(). 271 * __GFP_WAIT being passed to INIT_RADIX_TREE().
266 */ 272 */
267int radix_tree_preload(gfp_t gfp_mask) 273static int __radix_tree_preload(gfp_t gfp_mask)
268{ 274{
269 struct radix_tree_preload *rtp; 275 struct radix_tree_preload *rtp;
270 struct radix_tree_node *node; 276 struct radix_tree_node *node;
@@ -288,9 +294,40 @@ int radix_tree_preload(gfp_t gfp_mask)
288out: 294out:
289 return ret; 295 return ret;
290} 296}
297
298/*
299 * Load up this CPU's radix_tree_node buffer with sufficient objects to
300 * ensure that the addition of a single element in the tree cannot fail. On
301 * success, return zero, with preemption disabled. On error, return -ENOMEM
302 * with preemption not disabled.
303 *
304 * To make use of this facility, the radix tree must be initialised without
305 * __GFP_WAIT being passed to INIT_RADIX_TREE().
306 */
307int radix_tree_preload(gfp_t gfp_mask)
308{
309 /* Warn on non-sensical use... */
310 WARN_ON_ONCE(!(gfp_mask & __GFP_WAIT));
311 return __radix_tree_preload(gfp_mask);
312}
291EXPORT_SYMBOL(radix_tree_preload); 313EXPORT_SYMBOL(radix_tree_preload);
292 314
293/* 315/*
316 * The same as above function, except we don't guarantee preloading happens.
317 * We do it, if we decide it helps. On success, return zero with preemption
318 * disabled. On error, return -ENOMEM with preemption not disabled.
319 */
320int radix_tree_maybe_preload(gfp_t gfp_mask)
321{
322 if (gfp_mask & __GFP_WAIT)
323 return __radix_tree_preload(gfp_mask);
324 /* Preloading doesn't help anything with this gfp mask, skip it */
325 preempt_disable();
326 return 0;
327}
328EXPORT_SYMBOL(radix_tree_maybe_preload);
329
330/*
294 * Return the maximum key which can be store into a 331 * Return the maximum key which can be store into a
295 * radix tree with height HEIGHT. 332 * radix tree with height HEIGHT.
296 */ 333 */
diff --git a/lib/raid6/.gitignore b/lib/raid6/.gitignore
index 162becacf97c..0a7e494b2bcd 100644
--- a/lib/raid6/.gitignore
+++ b/lib/raid6/.gitignore
@@ -2,3 +2,4 @@ mktables
2altivec*.c 2altivec*.c
3int*.c 3int*.c
4tables.c 4tables.c
5neon?.c
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 9f7c184725d7..c7dab0645554 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -5,6 +5,8 @@ raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \
5 5
6raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o 6raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o
7raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o 7raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o
8raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o
9raid6_pq-$(CONFIG_TILEGX) += tilegx8.o
8 10
9hostprogs-y += mktables 11hostprogs-y += mktables
10 12
@@ -16,6 +18,21 @@ ifeq ($(CONFIG_ALTIVEC),y)
16altivec_flags := -maltivec -mabi=altivec 18altivec_flags := -maltivec -mabi=altivec
17endif 19endif
18 20
21# The GCC option -ffreestanding is required in order to compile code containing
22# ARM/NEON intrinsics in a non C99-compliant environment (such as the kernel)
23ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
24NEON_FLAGS := -ffreestanding
25ifeq ($(ARCH),arm)
26NEON_FLAGS += -mfloat-abi=softfp -mfpu=neon
27endif
28ifeq ($(ARCH),arm64)
29CFLAGS_REMOVE_neon1.o += -mgeneral-regs-only
30CFLAGS_REMOVE_neon2.o += -mgeneral-regs-only
31CFLAGS_REMOVE_neon4.o += -mgeneral-regs-only
32CFLAGS_REMOVE_neon8.o += -mgeneral-regs-only
33endif
34endif
35
19targets += int1.c 36targets += int1.c
20$(obj)/int1.c: UNROLL := 1 37$(obj)/int1.c: UNROLL := 1
21$(obj)/int1.c: $(src)/int.uc $(src)/unroll.awk FORCE 38$(obj)/int1.c: $(src)/int.uc $(src)/unroll.awk FORCE
@@ -70,6 +87,35 @@ $(obj)/altivec8.c: UNROLL := 8
70$(obj)/altivec8.c: $(src)/altivec.uc $(src)/unroll.awk FORCE 87$(obj)/altivec8.c: $(src)/altivec.uc $(src)/unroll.awk FORCE
71 $(call if_changed,unroll) 88 $(call if_changed,unroll)
72 89
90CFLAGS_neon1.o += $(NEON_FLAGS)
91targets += neon1.c
92$(obj)/neon1.c: UNROLL := 1
93$(obj)/neon1.c: $(src)/neon.uc $(src)/unroll.awk FORCE
94 $(call if_changed,unroll)
95
96CFLAGS_neon2.o += $(NEON_FLAGS)
97targets += neon2.c
98$(obj)/neon2.c: UNROLL := 2
99$(obj)/neon2.c: $(src)/neon.uc $(src)/unroll.awk FORCE
100 $(call if_changed,unroll)
101
102CFLAGS_neon4.o += $(NEON_FLAGS)
103targets += neon4.c
104$(obj)/neon4.c: UNROLL := 4
105$(obj)/neon4.c: $(src)/neon.uc $(src)/unroll.awk FORCE
106 $(call if_changed,unroll)
107
108CFLAGS_neon8.o += $(NEON_FLAGS)
109targets += neon8.c
110$(obj)/neon8.c: UNROLL := 8
111$(obj)/neon8.c: $(src)/neon.uc $(src)/unroll.awk FORCE
112 $(call if_changed,unroll)
113
114targets += tilegx8.c
115$(obj)/tilegx8.c: UNROLL := 8
116$(obj)/tilegx8.c: $(src)/tilegx.uc $(src)/unroll.awk FORCE
117 $(call if_changed,unroll)
118
73quiet_cmd_mktable = TABLE $@ 119quiet_cmd_mktable = TABLE $@
74 cmd_mktable = $(obj)/mktables > $@ || ( rm -f $@ && exit 1 ) 120 cmd_mktable = $(obj)/mktables > $@ || ( rm -f $@ && exit 1 )
75 121
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 6d7316fe9f30..f0b1aa3586d1 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -66,10 +66,19 @@ const struct raid6_calls * const raid6_algos[] = {
66 &raid6_altivec4, 66 &raid6_altivec4,
67 &raid6_altivec8, 67 &raid6_altivec8,
68#endif 68#endif
69#if defined(CONFIG_TILEGX)
70 &raid6_tilegx8,
71#endif
69 &raid6_intx1, 72 &raid6_intx1,
70 &raid6_intx2, 73 &raid6_intx2,
71 &raid6_intx4, 74 &raid6_intx4,
72 &raid6_intx8, 75 &raid6_intx8,
76#ifdef CONFIG_KERNEL_MODE_NEON
77 &raid6_neonx1,
78 &raid6_neonx2,
79 &raid6_neonx4,
80 &raid6_neonx8,
81#endif
73 NULL 82 NULL
74}; 83};
75 84
diff --git a/lib/raid6/neon.c b/lib/raid6/neon.c
new file mode 100644
index 000000000000..36ad4705df1a
--- /dev/null
+++ b/lib/raid6/neon.c
@@ -0,0 +1,58 @@
1/*
2 * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics
3 *
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/raid/pq.h>
12
13#ifdef __KERNEL__
14#include <asm/neon.h>
15#else
16#define kernel_neon_begin()
17#define kernel_neon_end()
18#define cpu_has_neon() (1)
19#endif
20
21/*
22 * There are 2 reasons these wrappers are kept in a separate compilation unit
23 * from the actual implementations in neonN.c (generated from neon.uc by
24 * unroll.awk):
25 * - the actual implementations use NEON intrinsics, and the GCC support header
26 * (arm_neon.h) is not fully compatible (type wise) with the kernel;
27 * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
28 * and we have to make sure that we never use *any* NEON/VFP instructions
29 * outside a kernel_neon_begin()/kernel_neon_end() pair.
30 */
31
32#define RAID6_NEON_WRAPPER(_n) \
33 static void raid6_neon ## _n ## _gen_syndrome(int disks, \
34 size_t bytes, void **ptrs) \
35 { \
36 void raid6_neon ## _n ## _gen_syndrome_real(int, \
37 unsigned long, void**); \
38 kernel_neon_begin(); \
39 raid6_neon ## _n ## _gen_syndrome_real(disks, \
40 (unsigned long)bytes, ptrs); \
41 kernel_neon_end(); \
42 } \
43 struct raid6_calls const raid6_neonx ## _n = { \
44 raid6_neon ## _n ## _gen_syndrome, \
45 raid6_have_neon, \
46 "neonx" #_n, \
47 0 \
48 }
49
50static int raid6_have_neon(void)
51{
52 return cpu_has_neon();
53}
54
55RAID6_NEON_WRAPPER(1);
56RAID6_NEON_WRAPPER(2);
57RAID6_NEON_WRAPPER(4);
58RAID6_NEON_WRAPPER(8);
diff --git a/lib/raid6/neon.uc b/lib/raid6/neon.uc
new file mode 100644
index 000000000000..1b9ed793342d
--- /dev/null
+++ b/lib/raid6/neon.uc
@@ -0,0 +1,80 @@
1/* -----------------------------------------------------------------------
2 *
3 * neon.uc - RAID-6 syndrome calculation using ARM NEON instructions
4 *
5 * Copyright (C) 2012 Rob Herring
6 *
7 * Based on altivec.uc:
8 * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
13 * Boston MA 02111-1307, USA; either version 2 of the License, or
14 * (at your option) any later version; incorporated herein by reference.
15 *
16 * ----------------------------------------------------------------------- */
17
18/*
19 * neon$#.c
20 *
21 * $#-way unrolled NEON intrinsics math RAID-6 instruction set
22 *
23 * This file is postprocessed using unroll.awk
24 */
25
26#include <arm_neon.h>
27
28typedef uint8x16_t unative_t;
29
30#define NBYTES(x) ((unative_t){x,x,x,x, x,x,x,x, x,x,x,x, x,x,x,x})
31#define NSIZE sizeof(unative_t)
32
33/*
34 * The SHLBYTE() operation shifts each byte left by 1, *not*
35 * rolling over into the next byte
36 */
37static inline unative_t SHLBYTE(unative_t v)
38{
39 return vshlq_n_u8(v, 1);
40}
41
42/*
43 * The MASK() operation returns 0xFF in any byte for which the high
44 * bit is 1, 0x00 for any byte for which the high bit is 0.
45 */
46static inline unative_t MASK(unative_t v)
47{
48 const uint8x16_t temp = NBYTES(0);
49 return (unative_t)vcltq_s8((int8x16_t)v, (int8x16_t)temp);
50}
51
52void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
53{
54 uint8_t **dptr = (uint8_t **)ptrs;
55 uint8_t *p, *q;
56 int d, z, z0;
57
58 register unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
59 const unative_t x1d = NBYTES(0x1d);
60
61 z0 = disks - 3; /* Highest data disk */
62 p = dptr[z0+1]; /* XOR parity */
63 q = dptr[z0+2]; /* RS syndrome */
64
65 for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
66 wq$$ = wp$$ = vld1q_u8(&dptr[z0][d+$$*NSIZE]);
67 for ( z = z0-1 ; z >= 0 ; z-- ) {
68 wd$$ = vld1q_u8(&dptr[z][d+$$*NSIZE]);
69 wp$$ = veorq_u8(wp$$, wd$$);
70 w2$$ = MASK(wq$$);
71 w1$$ = SHLBYTE(wq$$);
72
73 w2$$ = vandq_u8(w2$$, x1d);
74 w1$$ = veorq_u8(w1$$, w2$$);
75 wq$$ = veorq_u8(w1$$, wd$$);
76 }
77 vst1q_u8(&p[d+NSIZE*$$], wp$$);
78 vst1q_u8(&q[d+NSIZE*$$], wq$$);
79 }
80}
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
index 087332dbf8aa..29090f3db677 100644
--- a/lib/raid6/test/Makefile
+++ b/lib/raid6/test/Makefile
@@ -22,19 +22,34 @@ ifeq ($(ARCH),x86_64)
22 IS_X86 = yes 22 IS_X86 = yes
23endif 23endif
24 24
25ifeq ($(ARCH),arm)
26 CFLAGS += -I../../../arch/arm/include -mfpu=neon
27 HAS_NEON = yes
28endif
29ifeq ($(ARCH),arm64)
30 CFLAGS += -I../../../arch/arm64/include
31 HAS_NEON = yes
32endif
33
25ifeq ($(IS_X86),yes) 34ifeq ($(IS_X86),yes)
26 OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o 35 OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o
27 CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \ 36 CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \
28 gcc -c -x assembler - >&/dev/null && \ 37 gcc -c -x assembler - >&/dev/null && \
29 rm ./-.o && echo -DCONFIG_AS_AVX2=1) 38 rm ./-.o && echo -DCONFIG_AS_AVX2=1)
39else ifeq ($(HAS_NEON),yes)
40 OBJS += neon.o neon1.o neon2.o neon4.o neon8.o
41 CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1
30else 42else
31 HAS_ALTIVEC := $(shell echo -e '\#include <altivec.h>\nvector int a;' |\ 43 HAS_ALTIVEC := $(shell printf '\#include <altivec.h>\nvector int a;\n' |\
32 gcc -c -x c - >&/dev/null && \ 44 gcc -c -x c - >&/dev/null && \
33 rm ./-.o && echo yes) 45 rm ./-.o && echo yes)
34 ifeq ($(HAS_ALTIVEC),yes) 46 ifeq ($(HAS_ALTIVEC),yes)
35 OBJS += altivec1.o altivec2.o altivec4.o altivec8.o 47 OBJS += altivec1.o altivec2.o altivec4.o altivec8.o
36 endif 48 endif
37endif 49endif
50ifeq ($(ARCH),tilegx)
51OBJS += tilegx8.o
52endif
38 53
39.c.o: 54.c.o:
40 $(CC) $(CFLAGS) -c -o $@ $< 55 $(CC) $(CFLAGS) -c -o $@ $<
@@ -55,6 +70,18 @@ raid6.a: $(OBJS)
55raid6test: test.c raid6.a 70raid6test: test.c raid6.a
56 $(CC) $(CFLAGS) -o raid6test $^ 71 $(CC) $(CFLAGS) -o raid6test $^
57 72
73neon1.c: neon.uc ../unroll.awk
74 $(AWK) ../unroll.awk -vN=1 < neon.uc > $@
75
76neon2.c: neon.uc ../unroll.awk
77 $(AWK) ../unroll.awk -vN=2 < neon.uc > $@
78
79neon4.c: neon.uc ../unroll.awk
80 $(AWK) ../unroll.awk -vN=4 < neon.uc > $@
81
82neon8.c: neon.uc ../unroll.awk
83 $(AWK) ../unroll.awk -vN=8 < neon.uc > $@
84
58altivec1.c: altivec.uc ../unroll.awk 85altivec1.c: altivec.uc ../unroll.awk
59 $(AWK) ../unroll.awk -vN=1 < altivec.uc > $@ 86 $(AWK) ../unroll.awk -vN=1 < altivec.uc > $@
60 87
@@ -85,11 +112,15 @@ int16.c: int.uc ../unroll.awk
85int32.c: int.uc ../unroll.awk 112int32.c: int.uc ../unroll.awk
86 $(AWK) ../unroll.awk -vN=32 < int.uc > $@ 113 $(AWK) ../unroll.awk -vN=32 < int.uc > $@
87 114
115tilegx8.c: tilegx.uc ../unroll.awk
116 $(AWK) ../unroll.awk -vN=8 < tilegx.uc > $@
117
88tables.c: mktables 118tables.c: mktables
89 ./mktables > tables.c 119 ./mktables > tables.c
90 120
91clean: 121clean:
92 rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c tables.c raid6test 122 rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c neon*.c tables.c raid6test
123 rm -f tilegx*.c
93 124
94spotless: clean 125spotless: clean
95 rm -f *~ 126 rm -f *~
diff --git a/lib/raid6/tilegx.uc b/lib/raid6/tilegx.uc
new file mode 100644
index 000000000000..e7c29459cbcd
--- /dev/null
+++ b/lib/raid6/tilegx.uc
@@ -0,0 +1,86 @@
1/* -*- linux-c -*- ------------------------------------------------------- *
2 *
3 * Copyright 2002 H. Peter Anvin - All Rights Reserved
4 * Copyright 2012 Tilera Corporation - All Rights Reserved
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
9 * Boston MA 02111-1307, USA; either version 2 of the License, or
10 * (at your option) any later version; incorporated herein by reference.
11 *
12 * ----------------------------------------------------------------------- */
13
14/*
15 * tilegx$#.c
16 *
17 * $#-way unrolled TILE-Gx SIMD for RAID-6 math.
18 *
19 * This file is postprocessed using unroll.awk.
20 *
21 */
22
23#include <linux/raid/pq.h>
24
25/* Create 8 byte copies of constant byte */
26# define NBYTES(x) (__insn_v1addi(0, x))
27# define NSIZE 8
28
29/*
30 * The SHLBYTE() operation shifts each byte left by 1, *not*
31 * rolling over into the next byte
32 */
33static inline __attribute_const__ u64 SHLBYTE(u64 v)
34{
35 /* Vector One Byte Shift Left Immediate. */
36 return __insn_v1shli(v, 1);
37}
38
39/*
40 * The MASK() operation returns 0xFF in any byte for which the high
41 * bit is 1, 0x00 for any byte for which the high bit is 0.
42 */
43static inline __attribute_const__ u64 MASK(u64 v)
44{
45 /* Vector One Byte Shift Right Signed Immediate. */
46 return __insn_v1shrsi(v, 7);
47}
48
49
50void raid6_tilegx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
51{
52 u8 **dptr = (u8 **)ptrs;
53 u64 *p, *q;
54 int d, z, z0;
55
56 u64 wd$$, wq$$, wp$$, w1$$, w2$$;
57 u64 x1d = NBYTES(0x1d);
58 u64 * z0ptr;
59
60 z0 = disks - 3; /* Highest data disk */
61 p = (u64 *)dptr[z0+1]; /* XOR parity */
62 q = (u64 *)dptr[z0+2]; /* RS syndrome */
63
64 z0ptr = (u64 *)&dptr[z0][0];
65 for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
66 wq$$ = wp$$ = *z0ptr++;
67 for ( z = z0-1 ; z >= 0 ; z-- ) {
68 wd$$ = *(u64 *)&dptr[z][d+$$*NSIZE];
69 wp$$ = wp$$ ^ wd$$;
70 w2$$ = MASK(wq$$);
71 w1$$ = SHLBYTE(wq$$);
72 w2$$ = w2$$ & x1d;
73 w1$$ = w1$$ ^ w2$$;
74 wq$$ = w1$$ ^ wd$$;
75 }
76 *p++ = wp$$;
77 *q++ = wq$$;
78 }
79}
80
81const struct raid6_calls raid6_tilegx$# = {
82 raid6_tilegx$#_gen_syndrome,
83 NULL,
84 "tilegx$#",
85 0
86};
diff --git a/lib/rbtree.c b/lib/rbtree.c
index c0e31fe2fabf..65f4effd117f 100644
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -518,3 +518,43 @@ void rb_replace_node(struct rb_node *victim, struct rb_node *new,
518 *new = *victim; 518 *new = *victim;
519} 519}
520EXPORT_SYMBOL(rb_replace_node); 520EXPORT_SYMBOL(rb_replace_node);
521
522static struct rb_node *rb_left_deepest_node(const struct rb_node *node)
523{
524 for (;;) {
525 if (node->rb_left)
526 node = node->rb_left;
527 else if (node->rb_right)
528 node = node->rb_right;
529 else
530 return (struct rb_node *)node;
531 }
532}
533
534struct rb_node *rb_next_postorder(const struct rb_node *node)
535{
536 const struct rb_node *parent;
537 if (!node)
538 return NULL;
539 parent = rb_parent(node);
540
541 /* If we're sitting on node, we've already seen our children */
542 if (parent && node == parent->rb_left && parent->rb_right) {
543 /* If we are the parent's left node, go to the parent's right
544 * node then all the way down to the left */
545 return rb_left_deepest_node(parent->rb_right);
546 } else
547 /* Otherwise we are the parent's right node, and the parent
548 * should be next */
549 return (struct rb_node *)parent;
550}
551EXPORT_SYMBOL(rb_next_postorder);
552
553struct rb_node *rb_first_postorder(const struct rb_root *root)
554{
555 if (!root->rb_node)
556 return NULL;
557
558 return rb_left_deepest_node(root->rb_node);
559}
560EXPORT_SYMBOL(rb_first_postorder);
diff --git a/lib/rbtree_test.c b/lib/rbtree_test.c
index 122f02f9941b..31dd4ccd3baa 100644
--- a/lib/rbtree_test.c
+++ b/lib/rbtree_test.c
@@ -114,6 +114,16 @@ static int black_path_count(struct rb_node *rb)
114 return count; 114 return count;
115} 115}
116 116
117static void check_postorder(int nr_nodes)
118{
119 struct rb_node *rb;
120 int count = 0;
121 for (rb = rb_first_postorder(&root); rb; rb = rb_next_postorder(rb))
122 count++;
123
124 WARN_ON_ONCE(count != nr_nodes);
125}
126
117static void check(int nr_nodes) 127static void check(int nr_nodes)
118{ 128{
119 struct rb_node *rb; 129 struct rb_node *rb;
@@ -136,6 +146,8 @@ static void check(int nr_nodes)
136 146
137 WARN_ON_ONCE(count != nr_nodes); 147 WARN_ON_ONCE(count != nr_nodes);
138 WARN_ON_ONCE(count < (1 << black_path_count(rb_last(&root))) - 1); 148 WARN_ON_ONCE(count < (1 << black_path_count(rb_last(&root))) - 1);
149
150 check_postorder(nr_nodes);
139} 151}
140 152
141static void check_augmented(int nr_nodes) 153static void check_augmented(int nr_nodes)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index d23762e6652c..4e8686c7e5a4 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -870,13 +870,13 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
870 swiotlb_full(hwdev, sg->length, dir, 0); 870 swiotlb_full(hwdev, sg->length, dir, 0);
871 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, 871 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
872 attrs); 872 attrs);
873 sgl[0].dma_length = 0; 873 sg_dma_len(sgl) = 0;
874 return 0; 874 return 0;
875 } 875 }
876 sg->dma_address = phys_to_dma(hwdev, map); 876 sg->dma_address = phys_to_dma(hwdev, map);
877 } else 877 } else
878 sg->dma_address = dev_addr; 878 sg->dma_address = dev_addr;
879 sg->dma_length = sg->length; 879 sg_dma_len(sg) = sg->length;
880 } 880 }
881 return nelems; 881 return nelems;
882} 882}
@@ -904,7 +904,7 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
904 BUG_ON(dir == DMA_NONE); 904 BUG_ON(dir == DMA_NONE);
905 905
906 for_each_sg(sgl, sg, nelems, i) 906 for_each_sg(sgl, sg, nelems, i)
907 unmap_single(hwdev, sg->dma_address, sg->dma_length, dir); 907 unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir);
908 908
909} 909}
910EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); 910EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
@@ -934,7 +934,7 @@ swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
934 934
935 for_each_sg(sgl, sg, nelems, i) 935 for_each_sg(sgl, sg, nelems, i)
936 swiotlb_sync_single(hwdev, sg->dma_address, 936 swiotlb_sync_single(hwdev, sg->dma_address,
937 sg->dma_length, dir, target); 937 sg_dma_len(sg), dir, target);
938} 938}
939 939
940void 940void
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 739a36366b79..26559bdb4c49 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -26,6 +26,7 @@
26#include <linux/math64.h> 26#include <linux/math64.h>
27#include <linux/uaccess.h> 27#include <linux/uaccess.h>
28#include <linux/ioport.h> 28#include <linux/ioport.h>
29#include <linux/dcache.h>
29#include <net/addrconf.h> 30#include <net/addrconf.h>
30 31
31#include <asm/page.h> /* for PAGE_SIZE */ 32#include <asm/page.h> /* for PAGE_SIZE */
@@ -532,6 +533,81 @@ char *string(char *buf, char *end, const char *s, struct printf_spec spec)
532 return buf; 533 return buf;
533} 534}
534 535
536static void widen(char *buf, char *end, unsigned len, unsigned spaces)
537{
538 size_t size;
539 if (buf >= end) /* nowhere to put anything */
540 return;
541 size = end - buf;
542 if (size <= spaces) {
543 memset(buf, ' ', size);
544 return;
545 }
546 if (len) {
547 if (len > size - spaces)
548 len = size - spaces;
549 memmove(buf + spaces, buf, len);
550 }
551 memset(buf, ' ', spaces);
552}
553
554static noinline_for_stack
555char *dentry_name(char *buf, char *end, const struct dentry *d, struct printf_spec spec,
556 const char *fmt)
557{
558 const char *array[4], *s;
559 const struct dentry *p;
560 int depth;
561 int i, n;
562
563 switch (fmt[1]) {
564 case '2': case '3': case '4':
565 depth = fmt[1] - '0';
566 break;
567 default:
568 depth = 1;
569 }
570
571 rcu_read_lock();
572 for (i = 0; i < depth; i++, d = p) {
573 p = ACCESS_ONCE(d->d_parent);
574 array[i] = ACCESS_ONCE(d->d_name.name);
575 if (p == d) {
576 if (i)
577 array[i] = "";
578 i++;
579 break;
580 }
581 }
582 s = array[--i];
583 for (n = 0; n != spec.precision; n++, buf++) {
584 char c = *s++;
585 if (!c) {
586 if (!i)
587 break;
588 c = '/';
589 s = array[--i];
590 }
591 if (buf < end)
592 *buf = c;
593 }
594 rcu_read_unlock();
595 if (n < spec.field_width) {
596 /* we want to pad the sucker */
597 unsigned spaces = spec.field_width - n;
598 if (!(spec.flags & LEFT)) {
599 widen(buf - n, end, n, spaces);
600 return buf + spaces;
601 }
602 while (spaces--) {
603 if (buf < end)
604 *buf = ' ';
605 ++buf;
606 }
607 }
608 return buf;
609}
610
535static noinline_for_stack 611static noinline_for_stack
536char *symbol_string(char *buf, char *end, void *ptr, 612char *symbol_string(char *buf, char *end, void *ptr,
537 struct printf_spec spec, const char *fmt) 613 struct printf_spec spec, const char *fmt)
@@ -1253,6 +1329,12 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
1253 spec.base = 16; 1329 spec.base = 16;
1254 return number(buf, end, 1330 return number(buf, end,
1255 (unsigned long long) *((phys_addr_t *)ptr), spec); 1331 (unsigned long long) *((phys_addr_t *)ptr), spec);
1332 case 'd':
1333 return dentry_name(buf, end, ptr, spec, fmt);
1334 case 'D':
1335 return dentry_name(buf, end,
1336 ((const struct file *)ptr)->f_path.dentry,
1337 spec, fmt);
1256 } 1338 }
1257 spec.flags |= SMALL; 1339 spec.flags |= SMALL;
1258 if (spec.field_width == -1) { 1340 if (spec.field_width == -1) {