aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig17
-rw-r--r--lib/Kconfig.debug42
-rw-r--r--lib/Makefile7
-rw-r--r--lib/bitmap.c12
-rw-r--r--lib/debugobjects.c11
-rw-r--r--lib/dma-debug.c10
-rw-r--r--lib/dynamic_debug.c190
-rw-r--r--lib/jedec_ddr_data.c135
-rw-r--r--lib/kobject.c14
-rw-r--r--lib/list_debug.c25
-rw-r--r--lib/radix-tree.c15
-rw-r--r--lib/raid6/Makefile2
-rw-r--r--lib/raid6/algos.c127
-rw-r--r--lib/raid6/mktables.c25
-rw-r--r--lib/raid6/recov.c15
-rw-r--r--lib/raid6/recov_ssse3.c335
-rw-r--r--lib/raid6/test/Makefile2
-rw-r--r--lib/raid6/test/test.c32
-rw-r--r--lib/raid6/x86.h15
-rw-r--r--lib/rational.c2
-rw-r--r--lib/spinlock_debug.c2
-rw-r--r--lib/stmp_device.c80
-rw-r--r--lib/string_helpers.c8
-rw-r--r--lib/strncpy_from_user.c113
-rw-r--r--lib/strnlen_user.c138
-rw-r--r--lib/swiotlb.c8
-rw-r--r--lib/test-kstrtox.c4
-rw-r--r--lib/vsprintf.c303
28 files changed, 1410 insertions, 279 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index 4a8aba2e5cc0..a9e15403434e 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -16,6 +16,12 @@ config BITREVERSE
16config RATIONAL 16config RATIONAL
17 boolean 17 boolean
18 18
19config GENERIC_STRNCPY_FROM_USER
20 bool
21
22config GENERIC_STRNLEN_USER
23 bool
24
19config GENERIC_FIND_FIRST_BIT 25config GENERIC_FIND_FIRST_BIT
20 bool 26 bool
21 27
@@ -33,6 +39,9 @@ config GENERIC_IO
33 boolean 39 boolean
34 default n 40 default n
35 41
42config STMP_DEVICE
43 bool
44
36config CRC_CCITT 45config CRC_CCITT
37 tristate "CRC-CCITT functions" 46 tristate "CRC-CCITT functions"
38 help 47 help
@@ -353,6 +362,14 @@ config CORDIC
353 This option provides an implementation of the CORDIC algorithm; 362 This option provides an implementation of the CORDIC algorithm;
354 calculations are in fixed point. Module will be called cordic. 363 calculations are in fixed point. Module will be called cordic.
355 364
365config DDR
366 bool "JEDEC DDR data"
367 help
368 Data from JEDEC specs for DDR SDRAM memories,
369 particularly the AC timing parameters and addressing
370 information. This data is useful for drivers handling
371 DDR SDRAM controllers.
372
356config MPILIB 373config MPILIB
357 tristate 374 tristate
358 select CLZ_TAB 375 select CLZ_TAB
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 6777153f18f3..a42d3ae39648 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -3,12 +3,16 @@ config PRINTK_TIME
3 bool "Show timing information on printks" 3 bool "Show timing information on printks"
4 depends on PRINTK 4 depends on PRINTK
5 help 5 help
6 Selecting this option causes timing information to be 6 Selecting this option causes time stamps of the printk()
7 included in printk output. This allows you to measure 7 messages to be added to the output of the syslog() system
8 the interval between kernel operations, including bootup 8 call and at the console.
9 operations. This is useful for identifying long delays 9
10 in kernel startup. Or add printk.time=1 at boot-time. 10 The timestamp is always recorded internally, and exported
11 See Documentation/kernel-parameters.txt 11 to /dev/kmsg. This flag just specifies if the timestamp should
12 be included, not that the timestamp is recorded.
13
14 The behavior is also controlled by the kernel command line
15 parameter printk.time=1. See Documentation/kernel-parameters.txt
12 16
13config DEFAULT_MESSAGE_LOGLEVEL 17config DEFAULT_MESSAGE_LOGLEVEL
14 int "Default message log level (1-7)" 18 int "Default message log level (1-7)"
@@ -70,6 +74,15 @@ config STRIP_ASM_SYMS
70 that look like '.Lxxx') so they don't pollute the output of 74 that look like '.Lxxx') so they don't pollute the output of
71 get_wchan() and suchlike. 75 get_wchan() and suchlike.
72 76
77config READABLE_ASM
78 bool "Generate readable assembler code"
79 depends on DEBUG_KERNEL
80 help
81 Disable some compiler optimizations that tend to generate human unreadable
82 assembler output. This may make the kernel slightly slower, but it helps
83 to keep kernel developers who have to stare a lot at assembler listings
84 sane.
85
73config UNUSED_SYMBOLS 86config UNUSED_SYMBOLS
74 bool "Enable unused/obsolete exported symbols" 87 bool "Enable unused/obsolete exported symbols"
75 default y if X86 88 default y if X86
@@ -1205,8 +1218,13 @@ config DYNAMIC_DEBUG
1205 otherwise be available at runtime. These messages can then be 1218 otherwise be available at runtime. These messages can then be
1206 enabled/disabled based on various levels of scope - per source file, 1219 enabled/disabled based on various levels of scope - per source file,
1207 function, module, format string, and line number. This mechanism 1220 function, module, format string, and line number. This mechanism
1208 implicitly enables all pr_debug() and dev_dbg() calls. The impact of 1221 implicitly compiles in all pr_debug() and dev_dbg() calls, which
1209 this compile option is a larger kernel text size of about 2%. 1222 enlarges the kernel text size by about 2%.
1223
1224 If a source file is compiled with DEBUG flag set, any
1225 pr_debug() calls in it are enabled by default, but can be
1226 disabled at runtime as below. Note that DEBUG flag is
1227 turned on by many CONFIG_*DEBUG* options.
1210 1228
1211 Usage: 1229 Usage:
1212 1230
@@ -1223,16 +1241,16 @@ config DYNAMIC_DEBUG
1223 lineno : line number of the debug statement 1241 lineno : line number of the debug statement
1224 module : module that contains the debug statement 1242 module : module that contains the debug statement
1225 function : function that contains the debug statement 1243 function : function that contains the debug statement
1226 flags : 'p' means the line is turned 'on' for printing 1244 flags : '=p' means the line is turned 'on' for printing
1227 format : the format used for the debug statement 1245 format : the format used for the debug statement
1228 1246
1229 From a live system: 1247 From a live system:
1230 1248
1231 nullarbor:~ # cat <debugfs>/dynamic_debug/control 1249 nullarbor:~ # cat <debugfs>/dynamic_debug/control
1232 # filename:lineno [module]function flags format 1250 # filename:lineno [module]function flags format
1233 fs/aio.c:222 [aio]__put_ioctx - "__put_ioctx:\040freeing\040%p\012" 1251 fs/aio.c:222 [aio]__put_ioctx =_ "__put_ioctx:\040freeing\040%p\012"
1234 fs/aio.c:248 [aio]ioctx_alloc - "ENOMEM:\040nr_events\040too\040high\012" 1252 fs/aio.c:248 [aio]ioctx_alloc =_ "ENOMEM:\040nr_events\040too\040high\012"
1235 fs/aio.c:1770 [aio]sys_io_cancel - "calling\040cancel\012" 1253 fs/aio.c:1770 [aio]sys_io_cancel =_ "calling\040cancel\012"
1236 1254
1237 Example usage: 1255 Example usage:
1238 1256
diff --git a/lib/Makefile b/lib/Makefile
index 18515f0267c4..8c31a0cb75e9 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -123,6 +123,13 @@ obj-$(CONFIG_SIGNATURE) += digsig.o
123 123
124obj-$(CONFIG_CLZ_TAB) += clz_tab.o 124obj-$(CONFIG_CLZ_TAB) += clz_tab.o
125 125
126obj-$(CONFIG_DDR) += jedec_ddr_data.o
127
128obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o
129obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o
130
131obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
132
126hostprogs-y := gen_crc32table 133hostprogs-y := gen_crc32table
127clean-files := crc32table.h 134clean-files := crc32table.h
128 135
diff --git a/lib/bitmap.c b/lib/bitmap.c
index b5a8b6ad2454..06fdfa1aeba7 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -369,7 +369,8 @@ EXPORT_SYMBOL(bitmap_find_next_zero_area);
369 * @nmaskbits: size of bitmap, in bits 369 * @nmaskbits: size of bitmap, in bits
370 * 370 *
371 * Exactly @nmaskbits bits are displayed. Hex digits are grouped into 371 * Exactly @nmaskbits bits are displayed. Hex digits are grouped into
372 * comma-separated sets of eight digits per set. 372 * comma-separated sets of eight digits per set. Returns the number of
373 * characters which were written to *buf, excluding the trailing \0.
373 */ 374 */
374int bitmap_scnprintf(char *buf, unsigned int buflen, 375int bitmap_scnprintf(char *buf, unsigned int buflen,
375 const unsigned long *maskp, int nmaskbits) 376 const unsigned long *maskp, int nmaskbits)
@@ -517,8 +518,8 @@ EXPORT_SYMBOL(bitmap_parse_user);
517 * 518 *
518 * Helper routine for bitmap_scnlistprintf(). Write decimal number 519 * Helper routine for bitmap_scnlistprintf(). Write decimal number
519 * or range to buf, suppressing output past buf+buflen, with optional 520 * or range to buf, suppressing output past buf+buflen, with optional
520 * comma-prefix. Return len of what would be written to buf, if it 521 * comma-prefix. Return len of what was written to *buf, excluding the
521 * all fit. 522 * trailing \0.
522 */ 523 */
523static inline int bscnl_emit(char *buf, int buflen, int rbot, int rtop, int len) 524static inline int bscnl_emit(char *buf, int buflen, int rbot, int rtop, int len)
524{ 525{
@@ -544,9 +545,8 @@ static inline int bscnl_emit(char *buf, int buflen, int rbot, int rtop, int len)
544 * the range. Output format is compatible with the format 545 * the range. Output format is compatible with the format
545 * accepted as input by bitmap_parselist(). 546 * accepted as input by bitmap_parselist().
546 * 547 *
547 * The return value is the number of characters which would be 548 * The return value is the number of characters which were written to *buf
548 * generated for the given input, excluding the trailing '\0', as 549 * excluding the trailing '\0', as per ISO C99's scnprintf.
549 * per ISO C99.
550 */ 550 */
551int bitmap_scnlistprintf(char *buf, unsigned int buflen, 551int bitmap_scnlistprintf(char *buf, unsigned int buflen,
552 const unsigned long *maskp, int nmaskbits) 552 const unsigned long *maskp, int nmaskbits)
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 0ab9ae8057f0..d11808ca4bc4 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -79,30 +79,29 @@ static const char *obj_states[ODEBUG_STATE_MAX] = {
79 [ODEBUG_STATE_NOTAVAILABLE] = "not available", 79 [ODEBUG_STATE_NOTAVAILABLE] = "not available",
80}; 80};
81 81
82static int fill_pool(void) 82static void fill_pool(void)
83{ 83{
84 gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN; 84 gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
85 struct debug_obj *new; 85 struct debug_obj *new;
86 unsigned long flags; 86 unsigned long flags;
87 87
88 if (likely(obj_pool_free >= ODEBUG_POOL_MIN_LEVEL)) 88 if (likely(obj_pool_free >= ODEBUG_POOL_MIN_LEVEL))
89 return obj_pool_free; 89 return;
90 90
91 if (unlikely(!obj_cache)) 91 if (unlikely(!obj_cache))
92 return obj_pool_free; 92 return;
93 93
94 while (obj_pool_free < ODEBUG_POOL_MIN_LEVEL) { 94 while (obj_pool_free < ODEBUG_POOL_MIN_LEVEL) {
95 95
96 new = kmem_cache_zalloc(obj_cache, gfp); 96 new = kmem_cache_zalloc(obj_cache, gfp);
97 if (!new) 97 if (!new)
98 return obj_pool_free; 98 return;
99 99
100 raw_spin_lock_irqsave(&pool_lock, flags); 100 raw_spin_lock_irqsave(&pool_lock, flags);
101 hlist_add_head(&new->node, &obj_pool); 101 hlist_add_head(&new->node, &obj_pool);
102 obj_pool_free++; 102 obj_pool_free++;
103 raw_spin_unlock_irqrestore(&pool_lock, flags); 103 raw_spin_unlock_irqrestore(&pool_lock, flags);
104 } 104 }
105 return obj_pool_free;
106} 105}
107 106
108/* 107/*
@@ -1052,10 +1051,10 @@ static int __init debug_objects_replace_static_objects(void)
1052 cnt++; 1051 cnt++;
1053 } 1052 }
1054 } 1053 }
1054 local_irq_enable();
1055 1055
1056 printk(KERN_DEBUG "ODEBUG: %d of %d active objects replaced\n", cnt, 1056 printk(KERN_DEBUG "ODEBUG: %d of %d active objects replaced\n", cnt,
1057 obj_pool_used); 1057 obj_pool_used);
1058 local_irq_enable();
1059 return 0; 1058 return 0;
1060free: 1059free:
1061 hlist_for_each_entry_safe(obj, node, tmp, &objects, node) { 1060 hlist_for_each_entry_safe(obj, node, tmp, &objects, node) {
diff --git a/lib/dma-debug.c b/lib/dma-debug.c
index 13ef2338be41..518aea714d21 100644
--- a/lib/dma-debug.c
+++ b/lib/dma-debug.c
@@ -430,7 +430,7 @@ static struct dma_debug_entry *__dma_entry_alloc(void)
430 */ 430 */
431static struct dma_debug_entry *dma_entry_alloc(void) 431static struct dma_debug_entry *dma_entry_alloc(void)
432{ 432{
433 struct dma_debug_entry *entry = NULL; 433 struct dma_debug_entry *entry;
434 unsigned long flags; 434 unsigned long flags;
435 435
436 spin_lock_irqsave(&free_entries_lock, flags); 436 spin_lock_irqsave(&free_entries_lock, flags);
@@ -438,11 +438,14 @@ static struct dma_debug_entry *dma_entry_alloc(void)
438 if (list_empty(&free_entries)) { 438 if (list_empty(&free_entries)) {
439 pr_err("DMA-API: debugging out of memory - disabling\n"); 439 pr_err("DMA-API: debugging out of memory - disabling\n");
440 global_disable = true; 440 global_disable = true;
441 goto out; 441 spin_unlock_irqrestore(&free_entries_lock, flags);
442 return NULL;
442 } 443 }
443 444
444 entry = __dma_entry_alloc(); 445 entry = __dma_entry_alloc();
445 446
447 spin_unlock_irqrestore(&free_entries_lock, flags);
448
446#ifdef CONFIG_STACKTRACE 449#ifdef CONFIG_STACKTRACE
447 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES; 450 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
448 entry->stacktrace.entries = entry->st_entries; 451 entry->stacktrace.entries = entry->st_entries;
@@ -450,9 +453,6 @@ static struct dma_debug_entry *dma_entry_alloc(void)
450 save_stack_trace(&entry->stacktrace); 453 save_stack_trace(&entry->stacktrace);
451#endif 454#endif
452 455
453out:
454 spin_unlock_irqrestore(&free_entries_lock, flags);
455
456 return entry; 456 return entry;
457} 457}
458 458
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index 310c753cf83e..7ca29a0a3019 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -107,20 +107,22 @@ static char *ddebug_describe_flags(struct _ddebug *dp, char *buf,
107 return buf; 107 return buf;
108} 108}
109 109
110#define vpr_info_dq(q, msg) \ 110#define vpr_info(fmt, ...) \
111do { \ 111 if (verbose) do { pr_info(fmt, ##__VA_ARGS__); } while (0)
112 if (verbose) \ 112
113 /* trim last char off format print */ \ 113#define vpr_info_dq(q, msg) \
114 pr_info("%s: func=\"%s\" file=\"%s\" " \ 114do { \
115 "module=\"%s\" format=\"%.*s\" " \ 115 /* trim last char off format print */ \
116 "lineno=%u-%u", \ 116 vpr_info("%s: func=\"%s\" file=\"%s\" " \
117 msg, \ 117 "module=\"%s\" format=\"%.*s\" " \
118 q->function ? q->function : "", \ 118 "lineno=%u-%u", \
119 q->filename ? q->filename : "", \ 119 msg, \
120 q->module ? q->module : "", \ 120 q->function ? q->function : "", \
121 (int)(q->format ? strlen(q->format) - 1 : 0), \ 121 q->filename ? q->filename : "", \
122 q->format ? q->format : "", \ 122 q->module ? q->module : "", \
123 q->first_lineno, q->last_lineno); \ 123 (int)(q->format ? strlen(q->format) - 1 : 0), \
124 q->format ? q->format : "", \
125 q->first_lineno, q->last_lineno); \
124} while (0) 126} while (0)
125 127
126/* 128/*
@@ -180,12 +182,11 @@ static int ddebug_change(const struct ddebug_query *query,
180 if (newflags == dp->flags) 182 if (newflags == dp->flags)
181 continue; 183 continue;
182 dp->flags = newflags; 184 dp->flags = newflags;
183 if (verbose) 185 vpr_info("changed %s:%d [%s]%s =%s\n",
184 pr_info("changed %s:%d [%s]%s =%s\n", 186 trim_prefix(dp->filename), dp->lineno,
185 trim_prefix(dp->filename), dp->lineno, 187 dt->mod_name, dp->function,
186 dt->mod_name, dp->function, 188 ddebug_describe_flags(dp, flagbuf,
187 ddebug_describe_flags(dp, flagbuf, 189 sizeof(flagbuf)));
188 sizeof(flagbuf)));
189 } 190 }
190 } 191 }
191 mutex_unlock(&ddebug_lock); 192 mutex_unlock(&ddebug_lock);
@@ -337,7 +338,7 @@ static int check_set(const char **dest, char *src, char *name)
337 * Returns 0 on success, <0 on error. 338 * Returns 0 on success, <0 on error.
338 */ 339 */
339static int ddebug_parse_query(char *words[], int nwords, 340static int ddebug_parse_query(char *words[], int nwords,
340 struct ddebug_query *query) 341 struct ddebug_query *query, const char *modname)
341{ 342{
342 unsigned int i; 343 unsigned int i;
343 int rc; 344 int rc;
@@ -347,6 +348,10 @@ static int ddebug_parse_query(char *words[], int nwords,
347 return -EINVAL; 348 return -EINVAL;
348 memset(query, 0, sizeof(*query)); 349 memset(query, 0, sizeof(*query));
349 350
351 if (modname)
352 /* support $modname.dyndbg=<multiple queries> */
353 query->module = modname;
354
350 for (i = 0 ; i < nwords ; i += 2) { 355 for (i = 0 ; i < nwords ; i += 2) {
351 if (!strcmp(words[i], "func")) 356 if (!strcmp(words[i], "func"))
352 rc = check_set(&query->function, words[i+1], "func"); 357 rc = check_set(&query->function, words[i+1], "func");
@@ -410,8 +415,7 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
410 default: 415 default:
411 return -EINVAL; 416 return -EINVAL;
412 } 417 }
413 if (verbose) 418 vpr_info("op='%c'\n", op);
414 pr_info("op='%c'\n", op);
415 419
416 for ( ; *str ; ++str) { 420 for ( ; *str ; ++str) {
417 for (i = ARRAY_SIZE(opt_array) - 1; i >= 0; i--) { 421 for (i = ARRAY_SIZE(opt_array) - 1; i >= 0; i--) {
@@ -423,8 +427,7 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
423 if (i < 0) 427 if (i < 0)
424 return -EINVAL; 428 return -EINVAL;
425 } 429 }
426 if (verbose) 430 vpr_info("flags=0x%x\n", flags);
427 pr_info("flags=0x%x\n", flags);
428 431
429 /* calculate final *flagsp, *maskp according to mask and op */ 432 /* calculate final *flagsp, *maskp according to mask and op */
430 switch (op) { 433 switch (op) {
@@ -441,12 +444,11 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
441 *flagsp = 0; 444 *flagsp = 0;
442 break; 445 break;
443 } 446 }
444 if (verbose) 447 vpr_info("*flagsp=0x%x *maskp=0x%x\n", *flagsp, *maskp);
445 pr_info("*flagsp=0x%x *maskp=0x%x\n", *flagsp, *maskp);
446 return 0; 448 return 0;
447} 449}
448 450
449static int ddebug_exec_query(char *query_string) 451static int ddebug_exec_query(char *query_string, const char *modname)
450{ 452{
451 unsigned int flags = 0, mask = 0; 453 unsigned int flags = 0, mask = 0;
452 struct ddebug_query query; 454 struct ddebug_query query;
@@ -457,7 +459,7 @@ static int ddebug_exec_query(char *query_string)
457 nwords = ddebug_tokenize(query_string, words, MAXWORDS); 459 nwords = ddebug_tokenize(query_string, words, MAXWORDS);
458 if (nwords <= 0) 460 if (nwords <= 0)
459 return -EINVAL; 461 return -EINVAL;
460 if (ddebug_parse_query(words, nwords-1, &query)) 462 if (ddebug_parse_query(words, nwords-1, &query, modname))
461 return -EINVAL; 463 return -EINVAL;
462 if (ddebug_parse_flags(words[nwords-1], &flags, &mask)) 464 if (ddebug_parse_flags(words[nwords-1], &flags, &mask))
463 return -EINVAL; 465 return -EINVAL;
@@ -473,7 +475,7 @@ static int ddebug_exec_query(char *query_string)
473 last error or number of matching callsites. Module name is either 475 last error or number of matching callsites. Module name is either
474 in param (for boot arg) or perhaps in query string. 476 in param (for boot arg) or perhaps in query string.
475*/ 477*/
476static int ddebug_exec_queries(char *query) 478static int ddebug_exec_queries(char *query, const char *modname)
477{ 479{
478 char *split; 480 char *split;
479 int i, errs = 0, exitcode = 0, rc, nfound = 0; 481 int i, errs = 0, exitcode = 0, rc, nfound = 0;
@@ -487,10 +489,9 @@ static int ddebug_exec_queries(char *query)
487 if (!query || !*query || *query == '#') 489 if (!query || !*query || *query == '#')
488 continue; 490 continue;
489 491
490 if (verbose) 492 vpr_info("query %d: \"%s\"\n", i, query);
491 pr_info("query %d: \"%s\"\n", i, query);
492 493
493 rc = ddebug_exec_query(query); 494 rc = ddebug_exec_query(query, modname);
494 if (rc < 0) { 495 if (rc < 0) {
495 errs++; 496 errs++;
496 exitcode = rc; 497 exitcode = rc;
@@ -498,7 +499,7 @@ static int ddebug_exec_queries(char *query)
498 nfound += rc; 499 nfound += rc;
499 i++; 500 i++;
500 } 501 }
501 pr_info("processed %d queries, with %d matches, %d errs\n", 502 vpr_info("processed %d queries, with %d matches, %d errs\n",
502 i, nfound, errs); 503 i, nfound, errs);
503 504
504 if (exitcode) 505 if (exitcode)
@@ -653,10 +654,9 @@ static ssize_t ddebug_proc_write(struct file *file, const char __user *ubuf,
653 return -EFAULT; 654 return -EFAULT;
654 } 655 }
655 tmpbuf[len] = '\0'; 656 tmpbuf[len] = '\0';
656 if (verbose) 657 vpr_info("read %d bytes from userspace\n", (int)len);
657 pr_info("read %d bytes from userspace\n", (int)len);
658 658
659 ret = ddebug_exec_queries(tmpbuf); 659 ret = ddebug_exec_queries(tmpbuf, NULL);
660 kfree(tmpbuf); 660 kfree(tmpbuf);
661 if (ret < 0) 661 if (ret < 0)
662 return ret; 662 return ret;
@@ -717,8 +717,7 @@ static void *ddebug_proc_start(struct seq_file *m, loff_t *pos)
717 struct _ddebug *dp; 717 struct _ddebug *dp;
718 int n = *pos; 718 int n = *pos;
719 719
720 if (verbose) 720 vpr_info("called m=%p *pos=%lld\n", m, (unsigned long long)*pos);
721 pr_info("called m=%p *pos=%lld\n", m, (unsigned long long)*pos);
722 721
723 mutex_lock(&ddebug_lock); 722 mutex_lock(&ddebug_lock);
724 723
@@ -742,9 +741,8 @@ static void *ddebug_proc_next(struct seq_file *m, void *p, loff_t *pos)
742 struct ddebug_iter *iter = m->private; 741 struct ddebug_iter *iter = m->private;
743 struct _ddebug *dp; 742 struct _ddebug *dp;
744 743
745 if (verbose) 744 vpr_info("called m=%p p=%p *pos=%lld\n",
746 pr_info("called m=%p p=%p *pos=%lld\n", 745 m, p, (unsigned long long)*pos);
747 m, p, (unsigned long long)*pos);
748 746
749 if (p == SEQ_START_TOKEN) 747 if (p == SEQ_START_TOKEN)
750 dp = ddebug_iter_first(iter); 748 dp = ddebug_iter_first(iter);
@@ -766,8 +764,7 @@ static int ddebug_proc_show(struct seq_file *m, void *p)
766 struct _ddebug *dp = p; 764 struct _ddebug *dp = p;
767 char flagsbuf[10]; 765 char flagsbuf[10];
768 766
769 if (verbose) 767 vpr_info("called m=%p p=%p\n", m, p);
770 pr_info("called m=%p p=%p\n", m, p);
771 768
772 if (p == SEQ_START_TOKEN) { 769 if (p == SEQ_START_TOKEN) {
773 seq_puts(m, 770 seq_puts(m,
@@ -791,8 +788,7 @@ static int ddebug_proc_show(struct seq_file *m, void *p)
791 */ 788 */
792static void ddebug_proc_stop(struct seq_file *m, void *p) 789static void ddebug_proc_stop(struct seq_file *m, void *p)
793{ 790{
794 if (verbose) 791 vpr_info("called m=%p p=%p\n", m, p);
795 pr_info("called m=%p p=%p\n", m, p);
796 mutex_unlock(&ddebug_lock); 792 mutex_unlock(&ddebug_lock);
797} 793}
798 794
@@ -815,8 +811,7 @@ static int ddebug_proc_open(struct inode *inode, struct file *file)
815 struct ddebug_iter *iter; 811 struct ddebug_iter *iter;
816 int err; 812 int err;
817 813
818 if (verbose) 814 vpr_info("called\n");
819 pr_info("called\n");
820 815
821 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 816 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
822 if (iter == NULL) 817 if (iter == NULL)
@@ -866,12 +861,51 @@ int ddebug_add_module(struct _ddebug *tab, unsigned int n,
866 list_add_tail(&dt->link, &ddebug_tables); 861 list_add_tail(&dt->link, &ddebug_tables);
867 mutex_unlock(&ddebug_lock); 862 mutex_unlock(&ddebug_lock);
868 863
869 if (verbose) 864 vpr_info("%u debug prints in module %s\n", n, dt->mod_name);
870 pr_info("%u debug prints in module %s\n", n, dt->mod_name);
871 return 0; 865 return 0;
872} 866}
873EXPORT_SYMBOL_GPL(ddebug_add_module); 867EXPORT_SYMBOL_GPL(ddebug_add_module);
874 868
869/* helper for ddebug_dyndbg_(boot|module)_param_cb */
870static int ddebug_dyndbg_param_cb(char *param, char *val,
871 const char *modname, int on_err)
872{
873 char *sep;
874
875 sep = strchr(param, '.');
876 if (sep) {
877 /* needed only for ddebug_dyndbg_boot_param_cb */
878 *sep = '\0';
879 modname = param;
880 param = sep + 1;
881 }
882 if (strcmp(param, "dyndbg"))
883 return on_err; /* determined by caller */
884
885 ddebug_exec_queries((val ? val : "+p"), modname);
886
887 return 0; /* query failure shouldnt stop module load */
888}
889
890/* handle both dyndbg and $module.dyndbg params at boot */
891static int ddebug_dyndbg_boot_param_cb(char *param, char *val,
892 const char *unused)
893{
894 vpr_info("%s=\"%s\"\n", param, val);
895 return ddebug_dyndbg_param_cb(param, val, NULL, 0);
896}
897
898/*
899 * modprobe foo finds foo.params in boot-args, strips "foo.", and
900 * passes them to load_module(). This callback gets unknown params,
901 * processes dyndbg params, rejects others.
902 */
903int ddebug_dyndbg_module_param_cb(char *param, char *val, const char *module)
904{
905 vpr_info("module: %s %s=\"%s\"\n", module, param, val);
906 return ddebug_dyndbg_param_cb(param, val, module, -ENOENT);
907}
908
875static void ddebug_table_free(struct ddebug_table *dt) 909static void ddebug_table_free(struct ddebug_table *dt)
876{ 910{
877 list_del_init(&dt->link); 911 list_del_init(&dt->link);
@@ -888,8 +922,7 @@ int ddebug_remove_module(const char *mod_name)
888 struct ddebug_table *dt, *nextdt; 922 struct ddebug_table *dt, *nextdt;
889 int ret = -ENOENT; 923 int ret = -ENOENT;
890 924
891 if (verbose) 925 vpr_info("removing module \"%s\"\n", mod_name);
892 pr_info("removing module \"%s\"\n", mod_name);
893 926
894 mutex_lock(&ddebug_lock); 927 mutex_lock(&ddebug_lock);
895 list_for_each_entry_safe(dt, nextdt, &ddebug_tables, link) { 928 list_for_each_entry_safe(dt, nextdt, &ddebug_tables, link) {
@@ -940,8 +973,10 @@ static int __init dynamic_debug_init(void)
940{ 973{
941 struct _ddebug *iter, *iter_start; 974 struct _ddebug *iter, *iter_start;
942 const char *modname = NULL; 975 const char *modname = NULL;
976 char *cmdline;
943 int ret = 0; 977 int ret = 0;
944 int n = 0; 978 int n = 0, entries = 0, modct = 0;
979 int verbose_bytes = 0;
945 980
946 if (__start___verbose == __stop___verbose) { 981 if (__start___verbose == __stop___verbose) {
947 pr_warn("_ddebug table is empty in a " 982 pr_warn("_ddebug table is empty in a "
@@ -952,10 +987,15 @@ static int __init dynamic_debug_init(void)
952 modname = iter->modname; 987 modname = iter->modname;
953 iter_start = iter; 988 iter_start = iter;
954 for (; iter < __stop___verbose; iter++) { 989 for (; iter < __stop___verbose; iter++) {
990 entries++;
991 verbose_bytes += strlen(iter->modname) + strlen(iter->function)
992 + strlen(iter->filename) + strlen(iter->format);
993
955 if (strcmp(modname, iter->modname)) { 994 if (strcmp(modname, iter->modname)) {
995 modct++;
956 ret = ddebug_add_module(iter_start, n, modname); 996 ret = ddebug_add_module(iter_start, n, modname);
957 if (ret) 997 if (ret)
958 goto out_free; 998 goto out_err;
959 n = 0; 999 n = 0;
960 modname = iter->modname; 1000 modname = iter->modname;
961 iter_start = iter; 1001 iter_start = iter;
@@ -964,29 +1004,45 @@ static int __init dynamic_debug_init(void)
964 } 1004 }
965 ret = ddebug_add_module(iter_start, n, modname); 1005 ret = ddebug_add_module(iter_start, n, modname);
966 if (ret) 1006 if (ret)
967 goto out_free; 1007 goto out_err;
1008
1009 ddebug_init_success = 1;
1010 vpr_info("%d modules, %d entries and %d bytes in ddebug tables,"
1011 " %d bytes in (readonly) verbose section\n",
1012 modct, entries, (int)( modct * sizeof(struct ddebug_table)),
1013 verbose_bytes + (int)(__stop___verbose - __start___verbose));
968 1014
969 /* ddebug_query boot param got passed -> set it up */ 1015 /* apply ddebug_query boot param, dont unload tables on err */
970 if (ddebug_setup_string[0] != '\0') { 1016 if (ddebug_setup_string[0] != '\0') {
971 ret = ddebug_exec_queries(ddebug_setup_string); 1017 pr_warn("ddebug_query param name is deprecated,"
1018 " change it to dyndbg\n");
1019 ret = ddebug_exec_queries(ddebug_setup_string, NULL);
972 if (ret < 0) 1020 if (ret < 0)
973 pr_warn("Invalid ddebug boot param %s", 1021 pr_warn("Invalid ddebug boot param %s",
974 ddebug_setup_string); 1022 ddebug_setup_string);
975 else 1023 else
976 pr_info("%d changes by ddebug_query\n", ret); 1024 pr_info("%d changes by ddebug_query\n", ret);
977
978 /* keep tables even on ddebug_query parse error */
979 ret = 0;
980 } 1025 }
1026 /* now that ddebug tables are loaded, process all boot args
1027 * again to find and activate queries given in dyndbg params.
1028 * While this has already been done for known boot params, it
1029 * ignored the unknown ones (dyndbg in particular). Reusing
1030 * parse_args avoids ad-hoc parsing. This will also attempt
1031 * to activate queries for not-yet-loaded modules, which is
1032 * slightly noisy if verbose, but harmless.
1033 */
1034 cmdline = kstrdup(saved_command_line, GFP_KERNEL);
1035 parse_args("dyndbg params", cmdline, NULL,
1036 0, 0, 0, &ddebug_dyndbg_boot_param_cb);
1037 kfree(cmdline);
1038 return 0;
981 1039
982out_free: 1040out_err:
983 if (ret) 1041 ddebug_remove_all_tables();
984 ddebug_remove_all_tables();
985 else
986 ddebug_init_success = 1;
987 return 0; 1042 return 0;
988} 1043}
989/* Allow early initialization for boot messages via boot param */ 1044/* Allow early initialization for boot messages via boot param */
990arch_initcall(dynamic_debug_init); 1045early_initcall(dynamic_debug_init);
1046
991/* Debugfs setup must be done later */ 1047/* Debugfs setup must be done later */
992module_init(dynamic_debug_init_debugfs); 1048fs_initcall(dynamic_debug_init_debugfs);
diff --git a/lib/jedec_ddr_data.c b/lib/jedec_ddr_data.c
new file mode 100644
index 000000000000..6d2cbf1d567f
--- /dev/null
+++ b/lib/jedec_ddr_data.c
@@ -0,0 +1,135 @@
1/*
2 * DDR addressing details and AC timing parameters from JEDEC specs
3 *
4 * Copyright (C) 2012 Texas Instruments, Inc.
5 *
6 * Aneesh V <aneesh@ti.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <memory/jedec_ddr.h>
14#include <linux/module.h>
15
16/* LPDDR2 addressing details from JESD209-2 section 2.4 */
17const struct lpddr2_addressing
18 lpddr2_jedec_addressing_table[NUM_DDR_ADDR_TABLE_ENTRIES] = {
19 {B4, T_REFI_15_6, T_RFC_90}, /* 64M */
20 {B4, T_REFI_15_6, T_RFC_90}, /* 128M */
21 {B4, T_REFI_7_8, T_RFC_90}, /* 256M */
22 {B4, T_REFI_7_8, T_RFC_90}, /* 512M */
23 {B8, T_REFI_7_8, T_RFC_130}, /* 1GS4 */
24 {B8, T_REFI_3_9, T_RFC_130}, /* 2GS4 */
25 {B8, T_REFI_3_9, T_RFC_130}, /* 4G */
26 {B8, T_REFI_3_9, T_RFC_210}, /* 8G */
27 {B4, T_REFI_7_8, T_RFC_130}, /* 1GS2 */
28 {B4, T_REFI_3_9, T_RFC_130}, /* 2GS2 */
29};
30EXPORT_SYMBOL_GPL(lpddr2_jedec_addressing_table);
31
32/* LPDDR2 AC timing parameters from JESD209-2 section 12 */
33const struct lpddr2_timings
34 lpddr2_jedec_timings[NUM_DDR_TIMING_TABLE_ENTRIES] = {
35 /* Speed bin 400(200 MHz) */
36 [0] = {
37 .max_freq = 200000000,
38 .min_freq = 10000000,
39 .tRPab = 21000,
40 .tRCD = 18000,
41 .tWR = 15000,
42 .tRAS_min = 42000,
43 .tRRD = 10000,
44 .tWTR = 10000,
45 .tXP = 7500,
46 .tRTP = 7500,
47 .tCKESR = 15000,
48 .tDQSCK_max = 5500,
49 .tFAW = 50000,
50 .tZQCS = 90000,
51 .tZQCL = 360000,
52 .tZQinit = 1000000,
53 .tRAS_max_ns = 70000,
54 .tDQSCK_max_derated = 6000,
55 },
56 /* Speed bin 533(266 MHz) */
57 [1] = {
58 .max_freq = 266666666,
59 .min_freq = 10000000,
60 .tRPab = 21000,
61 .tRCD = 18000,
62 .tWR = 15000,
63 .tRAS_min = 42000,
64 .tRRD = 10000,
65 .tWTR = 7500,
66 .tXP = 7500,
67 .tRTP = 7500,
68 .tCKESR = 15000,
69 .tDQSCK_max = 5500,
70 .tFAW = 50000,
71 .tZQCS = 90000,
72 .tZQCL = 360000,
73 .tZQinit = 1000000,
74 .tRAS_max_ns = 70000,
75 .tDQSCK_max_derated = 6000,
76 },
77 /* Speed bin 800(400 MHz) */
78 [2] = {
79 .max_freq = 400000000,
80 .min_freq = 10000000,
81 .tRPab = 21000,
82 .tRCD = 18000,
83 .tWR = 15000,
84 .tRAS_min = 42000,
85 .tRRD = 10000,
86 .tWTR = 7500,
87 .tXP = 7500,
88 .tRTP = 7500,
89 .tCKESR = 15000,
90 .tDQSCK_max = 5500,
91 .tFAW = 50000,
92 .tZQCS = 90000,
93 .tZQCL = 360000,
94 .tZQinit = 1000000,
95 .tRAS_max_ns = 70000,
96 .tDQSCK_max_derated = 6000,
97 },
98 /* Speed bin 1066(533 MHz) */
99 [3] = {
100 .max_freq = 533333333,
101 .min_freq = 10000000,
102 .tRPab = 21000,
103 .tRCD = 18000,
104 .tWR = 15000,
105 .tRAS_min = 42000,
106 .tRRD = 10000,
107 .tWTR = 7500,
108 .tXP = 7500,
109 .tRTP = 7500,
110 .tCKESR = 15000,
111 .tDQSCK_max = 5500,
112 .tFAW = 50000,
113 .tZQCS = 90000,
114 .tZQCL = 360000,
115 .tZQinit = 1000000,
116 .tRAS_max_ns = 70000,
117 .tDQSCK_max_derated = 5620,
118 },
119};
120EXPORT_SYMBOL_GPL(lpddr2_jedec_timings);
121
122const struct lpddr2_min_tck lpddr2_jedec_min_tck = {
123 .tRPab = 3,
124 .tRCD = 3,
125 .tWR = 3,
126 .tRASmin = 3,
127 .tRRD = 2,
128 .tWTR = 2,
129 .tXP = 2,
130 .tRTP = 2,
131 .tCKE = 3,
132 .tCKESR = 3,
133 .tFAW = 8
134};
135EXPORT_SYMBOL_GPL(lpddr2_jedec_min_tck);
diff --git a/lib/kobject.c b/lib/kobject.c
index aeefa8bc8b1c..e07ee1fcd6f1 100644
--- a/lib/kobject.c
+++ b/lib/kobject.c
@@ -47,13 +47,11 @@ static int populate_dir(struct kobject *kobj)
47static int create_dir(struct kobject *kobj) 47static int create_dir(struct kobject *kobj)
48{ 48{
49 int error = 0; 49 int error = 0;
50 if (kobject_name(kobj)) { 50 error = sysfs_create_dir(kobj);
51 error = sysfs_create_dir(kobj); 51 if (!error) {
52 if (!error) { 52 error = populate_dir(kobj);
53 error = populate_dir(kobj); 53 if (error)
54 if (error) 54 sysfs_remove_dir(kobj);
55 sysfs_remove_dir(kobj);
56 }
57 } 55 }
58 return error; 56 return error;
59} 57}
@@ -634,7 +632,7 @@ struct kobject *kobject_create(void)
634/** 632/**
635 * kobject_create_and_add - create a struct kobject dynamically and register it with sysfs 633 * kobject_create_and_add - create a struct kobject dynamically and register it with sysfs
636 * 634 *
637 * @name: the name for the kset 635 * @name: the name for the kobject
638 * @parent: the parent kobject of this kobject, if any. 636 * @parent: the parent kobject of this kobject, if any.
639 * 637 *
640 * This function creates a kobject structure dynamically and registers it 638 * This function creates a kobject structure dynamically and registers it
diff --git a/lib/list_debug.c b/lib/list_debug.c
index 982b850d4e7a..23a5e031cd8b 100644
--- a/lib/list_debug.c
+++ b/lib/list_debug.c
@@ -10,6 +10,7 @@
10#include <linux/list.h> 10#include <linux/list.h>
11#include <linux/bug.h> 11#include <linux/bug.h>
12#include <linux/kernel.h> 12#include <linux/kernel.h>
13#include <linux/rculist.h>
13 14
14/* 15/*
15 * Insert a new entry between two known consecutive entries. 16 * Insert a new entry between two known consecutive entries.
@@ -30,6 +31,9 @@ void __list_add(struct list_head *new,
30 "list_add corruption. prev->next should be " 31 "list_add corruption. prev->next should be "
31 "next (%p), but was %p. (prev=%p).\n", 32 "next (%p), but was %p. (prev=%p).\n",
32 next, prev->next, prev); 33 next, prev->next, prev);
34 WARN(new == prev || new == next,
35 "list_add double add: new=%p, prev=%p, next=%p.\n",
36 new, prev, next);
33 next->prev = new; 37 next->prev = new;
34 new->next = next; 38 new->next = next;
35 new->prev = prev; 39 new->prev = prev;
@@ -75,3 +79,24 @@ void list_del(struct list_head *entry)
75 entry->prev = LIST_POISON2; 79 entry->prev = LIST_POISON2;
76} 80}
77EXPORT_SYMBOL(list_del); 81EXPORT_SYMBOL(list_del);
82
83/*
84 * RCU variants.
85 */
86void __list_add_rcu(struct list_head *new,
87 struct list_head *prev, struct list_head *next)
88{
89 WARN(next->prev != prev,
90 "list_add_rcu corruption. next->prev should be "
91 "prev (%p), but was %p. (next=%p).\n",
92 prev, next->prev, next);
93 WARN(prev->next != next,
94 "list_add_rcu corruption. prev->next should be "
95 "next (%p), but was %p. (prev=%p).\n",
96 next, prev->next, prev);
97 new->next = next;
98 new->prev = prev;
99 rcu_assign_pointer(list_next_rcu(prev), new);
100 next->prev = new;
101}
102EXPORT_SYMBOL(__list_add_rcu);
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 86516f5588e3..d7c878cc006c 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -73,11 +73,24 @@ static unsigned long height_to_maxindex[RADIX_TREE_MAX_PATH + 1] __read_mostly;
73static struct kmem_cache *radix_tree_node_cachep; 73static struct kmem_cache *radix_tree_node_cachep;
74 74
75/* 75/*
76 * The radix tree is variable-height, so an insert operation not only has
77 * to build the branch to its corresponding item, it also has to build the
78 * branch to existing items if the size has to be increased (by
79 * radix_tree_extend).
80 *
81 * The worst case is a zero height tree with just a single item at index 0,
82 * and then inserting an item at index ULONG_MAX. This requires 2 new branches
83 * of RADIX_TREE_MAX_PATH size to be created, with only the root node shared.
84 * Hence:
85 */
86#define RADIX_TREE_PRELOAD_SIZE (RADIX_TREE_MAX_PATH * 2 - 1)
87
88/*
76 * Per-cpu pool of preloaded nodes 89 * Per-cpu pool of preloaded nodes
77 */ 90 */
78struct radix_tree_preload { 91struct radix_tree_preload {
79 int nr; 92 int nr;
80 struct radix_tree_node *nodes[RADIX_TREE_MAX_PATH]; 93 struct radix_tree_node *nodes[RADIX_TREE_PRELOAD_SIZE];
81}; 94};
82static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, }; 95static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, };
83 96
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 8a38102770f3..de06dfe165b8 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -1,6 +1,6 @@
1obj-$(CONFIG_RAID6_PQ) += raid6_pq.o 1obj-$(CONFIG_RAID6_PQ) += raid6_pq.o
2 2
3raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ 3raid6_pq-y += algos.o recov.o recov_ssse3.o tables.o int1.o int2.o int4.o \
4 int8.o int16.o int32.o altivec1.o altivec2.o altivec4.o \ 4 int8.o int16.o int32.o altivec1.o altivec2.o altivec4.o \
5 altivec8.o mmx.o sse1.o sse2.o 5 altivec8.o mmx.o sse1.o sse2.o
6hostprogs-y += mktables 6hostprogs-y += mktables
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 8b02f60ffc86..589f5f50ad2e 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -17,11 +17,11 @@
17 */ 17 */
18 18
19#include <linux/raid/pq.h> 19#include <linux/raid/pq.h>
20#include <linux/module.h>
21#ifndef __KERNEL__ 20#ifndef __KERNEL__
22#include <sys/mman.h> 21#include <sys/mman.h>
23#include <stdio.h> 22#include <stdio.h>
24#else 23#else
24#include <linux/module.h>
25#include <linux/gfp.h> 25#include <linux/gfp.h>
26#if !RAID6_USE_EMPTY_ZERO_PAGE 26#if !RAID6_USE_EMPTY_ZERO_PAGE
27/* In .bss so it's zeroed */ 27/* In .bss so it's zeroed */
@@ -34,10 +34,6 @@ struct raid6_calls raid6_call;
34EXPORT_SYMBOL_GPL(raid6_call); 34EXPORT_SYMBOL_GPL(raid6_call);
35 35
36const struct raid6_calls * const raid6_algos[] = { 36const struct raid6_calls * const raid6_algos[] = {
37 &raid6_intx1,
38 &raid6_intx2,
39 &raid6_intx4,
40 &raid6_intx8,
41#if defined(__ia64__) 37#if defined(__ia64__)
42 &raid6_intx16, 38 &raid6_intx16,
43 &raid6_intx32, 39 &raid6_intx32,
@@ -61,6 +57,24 @@ const struct raid6_calls * const raid6_algos[] = {
61 &raid6_altivec4, 57 &raid6_altivec4,
62 &raid6_altivec8, 58 &raid6_altivec8,
63#endif 59#endif
60 &raid6_intx1,
61 &raid6_intx2,
62 &raid6_intx4,
63 &raid6_intx8,
64 NULL
65};
66
67void (*raid6_2data_recov)(int, size_t, int, int, void **);
68EXPORT_SYMBOL_GPL(raid6_2data_recov);
69
70void (*raid6_datap_recov)(int, size_t, int, void **);
71EXPORT_SYMBOL_GPL(raid6_datap_recov);
72
73const struct raid6_recov_calls *const raid6_recov_algos[] = {
74#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__)
75 &raid6_recov_ssse3,
76#endif
77 &raid6_recov_intx1,
64 NULL 78 NULL
65}; 79};
66 80
@@ -72,59 +86,55 @@ const struct raid6_calls * const raid6_algos[] = {
72#define time_before(x, y) ((x) < (y)) 86#define time_before(x, y) ((x) < (y))
73#endif 87#endif
74 88
75/* Try to pick the best algorithm */ 89static inline const struct raid6_recov_calls *raid6_choose_recov(void)
76/* This code uses the gfmul table as convenient data set to abuse */
77
78int __init raid6_select_algo(void)
79{ 90{
80 const struct raid6_calls * const * algo; 91 const struct raid6_recov_calls *const *algo;
81 const struct raid6_calls * best; 92 const struct raid6_recov_calls *best;
82 char *syndromes;
83 void *dptrs[(65536/PAGE_SIZE)+2];
84 int i, disks;
85 unsigned long perf, bestperf;
86 int bestprefer;
87 unsigned long j0, j1;
88 93
89 disks = (65536/PAGE_SIZE)+2; 94 for (best = NULL, algo = raid6_recov_algos; *algo; algo++)
90 for ( i = 0 ; i < disks-2 ; i++ ) { 95 if (!best || (*algo)->priority > best->priority)
91 dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i; 96 if (!(*algo)->valid || (*algo)->valid())
92 } 97 best = *algo;
93 98
94 /* Normal code - use a 2-page allocation to avoid D$ conflict */ 99 if (best) {
95 syndromes = (void *) __get_free_pages(GFP_KERNEL, 1); 100 raid6_2data_recov = best->data2;
101 raid6_datap_recov = best->datap;
96 102
97 if ( !syndromes ) { 103 printk("raid6: using %s recovery algorithm\n", best->name);
98 printk("raid6: Yikes! No memory available.\n"); 104 } else
99 return -ENOMEM; 105 printk("raid6: Yikes! No recovery algorithm found!\n");
100 }
101 106
102 dptrs[disks-2] = syndromes; 107 return best;
103 dptrs[disks-1] = syndromes + PAGE_SIZE; 108}
109
110static inline const struct raid6_calls *raid6_choose_gen(
111 void *(*const dptrs)[(65536/PAGE_SIZE)+2], const int disks)
112{
113 unsigned long perf, bestperf, j0, j1;
114 const struct raid6_calls *const *algo;
115 const struct raid6_calls *best;
104 116
105 bestperf = 0; bestprefer = 0; best = NULL; 117 for (bestperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
118 if (!best || (*algo)->prefer >= best->prefer) {
119 if ((*algo)->valid && !(*algo)->valid())
120 continue;
106 121
107 for ( algo = raid6_algos ; *algo ; algo++ ) {
108 if ( !(*algo)->valid || (*algo)->valid() ) {
109 perf = 0; 122 perf = 0;
110 123
111 preempt_disable(); 124 preempt_disable();
112 j0 = jiffies; 125 j0 = jiffies;
113 while ( (j1 = jiffies) == j0 ) 126 while ((j1 = jiffies) == j0)
114 cpu_relax(); 127 cpu_relax();
115 while (time_before(jiffies, 128 while (time_before(jiffies,
116 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { 129 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
117 (*algo)->gen_syndrome(disks, PAGE_SIZE, dptrs); 130 (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs);
118 perf++; 131 perf++;
119 } 132 }
120 preempt_enable(); 133 preempt_enable();
121 134
122 if ( (*algo)->prefer > bestprefer || 135 if (perf > bestperf) {
123 ((*algo)->prefer == bestprefer &&
124 perf > bestperf) ) {
125 best = *algo;
126 bestprefer = best->prefer;
127 bestperf = perf; 136 bestperf = perf;
137 best = *algo;
128 } 138 }
129 printk("raid6: %-8s %5ld MB/s\n", (*algo)->name, 139 printk("raid6: %-8s %5ld MB/s\n", (*algo)->name,
130 (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2)); 140 (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
@@ -139,9 +149,46 @@ int __init raid6_select_algo(void)
139 } else 149 } else
140 printk("raid6: Yikes! No algorithm found!\n"); 150 printk("raid6: Yikes! No algorithm found!\n");
141 151
152 return best;
153}
154
155
156/* Try to pick the best algorithm */
157/* This code uses the gfmul table as convenient data set to abuse */
158
159int __init raid6_select_algo(void)
160{
161 const int disks = (65536/PAGE_SIZE)+2;
162
163 const struct raid6_calls *gen_best;
164 const struct raid6_recov_calls *rec_best;
165 char *syndromes;
166 void *dptrs[(65536/PAGE_SIZE)+2];
167 int i;
168
169 for (i = 0; i < disks-2; i++)
170 dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i;
171
172 /* Normal code - use a 2-page allocation to avoid D$ conflict */
173 syndromes = (void *) __get_free_pages(GFP_KERNEL, 1);
174
175 if (!syndromes) {
176 printk("raid6: Yikes! No memory available.\n");
177 return -ENOMEM;
178 }
179
180 dptrs[disks-2] = syndromes;
181 dptrs[disks-1] = syndromes + PAGE_SIZE;
182
183 /* select raid gen_syndrome function */
184 gen_best = raid6_choose_gen(&dptrs, disks);
185
186 /* select raid recover functions */
187 rec_best = raid6_choose_recov();
188
142 free_pages((unsigned long)syndromes, 1); 189 free_pages((unsigned long)syndromes, 1);
143 190
144 return best ? 0 : -EINVAL; 191 return gen_best && rec_best ? 0 : -EINVAL;
145} 192}
146 193
147static void raid6_exit(void) 194static void raid6_exit(void)
diff --git a/lib/raid6/mktables.c b/lib/raid6/mktables.c
index 8a3780902cec..39787db588b0 100644
--- a/lib/raid6/mktables.c
+++ b/lib/raid6/mktables.c
@@ -81,6 +81,31 @@ int main(int argc, char *argv[])
81 printf("EXPORT_SYMBOL(raid6_gfmul);\n"); 81 printf("EXPORT_SYMBOL(raid6_gfmul);\n");
82 printf("#endif\n"); 82 printf("#endif\n");
83 83
84 /* Compute vector multiplication table */
85 printf("\nconst u8 __attribute__((aligned(256)))\n"
86 "raid6_vgfmul[256][32] =\n"
87 "{\n");
88 for (i = 0; i < 256; i++) {
89 printf("\t{\n");
90 for (j = 0; j < 16; j += 8) {
91 printf("\t\t");
92 for (k = 0; k < 8; k++)
93 printf("0x%02x,%c", gfmul(i, j + k),
94 (k == 7) ? '\n' : ' ');
95 }
96 for (j = 0; j < 16; j += 8) {
97 printf("\t\t");
98 for (k = 0; k < 8; k++)
99 printf("0x%02x,%c", gfmul(i, (j + k) << 4),
100 (k == 7) ? '\n' : ' ');
101 }
102 printf("\t},\n");
103 }
104 printf("};\n");
105 printf("#ifdef __KERNEL__\n");
106 printf("EXPORT_SYMBOL(raid6_vgfmul);\n");
107 printf("#endif\n");
108
84 /* Compute power-of-2 table (exponent) */ 109 /* Compute power-of-2 table (exponent) */
85 v = 1; 110 v = 1;
86 printf("\nconst u8 __attribute__((aligned(256)))\n" 111 printf("\nconst u8 __attribute__((aligned(256)))\n"
diff --git a/lib/raid6/recov.c b/lib/raid6/recov.c
index fe275d7b6b36..1805a5cc5daa 100644
--- a/lib/raid6/recov.c
+++ b/lib/raid6/recov.c
@@ -22,7 +22,7 @@
22#include <linux/raid/pq.h> 22#include <linux/raid/pq.h>
23 23
24/* Recover two failed data blocks. */ 24/* Recover two failed data blocks. */
25void raid6_2data_recov(int disks, size_t bytes, int faila, int failb, 25void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, int failb,
26 void **ptrs) 26 void **ptrs)
27{ 27{
28 u8 *p, *q, *dp, *dq; 28 u8 *p, *q, *dp, *dq;
@@ -64,10 +64,9 @@ void raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
64 p++; q++; 64 p++; q++;
65 } 65 }
66} 66}
67EXPORT_SYMBOL_GPL(raid6_2data_recov);
68 67
69/* Recover failure of one data block plus the P block */ 68/* Recover failure of one data block plus the P block */
70void raid6_datap_recov(int disks, size_t bytes, int faila, void **ptrs) 69void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, void **ptrs)
71{ 70{
72 u8 *p, *q, *dq; 71 u8 *p, *q, *dq;
73 const u8 *qmul; /* Q multiplier table */ 72 const u8 *qmul; /* Q multiplier table */
@@ -96,7 +95,15 @@ void raid6_datap_recov(int disks, size_t bytes, int faila, void **ptrs)
96 q++; dq++; 95 q++; dq++;
97 } 96 }
98} 97}
99EXPORT_SYMBOL_GPL(raid6_datap_recov); 98
99
100const struct raid6_recov_calls raid6_recov_intx1 = {
101 .data2 = raid6_2data_recov_intx1,
102 .datap = raid6_datap_recov_intx1,
103 .valid = NULL,
104 .name = "intx1",
105 .priority = 0,
106};
100 107
101#ifndef __KERNEL__ 108#ifndef __KERNEL__
102/* Testing only */ 109/* Testing only */
diff --git a/lib/raid6/recov_ssse3.c b/lib/raid6/recov_ssse3.c
new file mode 100644
index 000000000000..37ae61930559
--- /dev/null
+++ b/lib/raid6/recov_ssse3.c
@@ -0,0 +1,335 @@
1/*
2 * Copyright (C) 2012 Intel Corporation
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; version 2
7 * of the License.
8 */
9
10#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__)
11
12#include <linux/raid/pq.h>
13#include "x86.h"
14
15static int raid6_has_ssse3(void)
16{
17 return boot_cpu_has(X86_FEATURE_XMM) &&
18 boot_cpu_has(X86_FEATURE_XMM2) &&
19 boot_cpu_has(X86_FEATURE_SSSE3);
20}
21
22void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, int failb,
23 void **ptrs)
24{
25 u8 *p, *q, *dp, *dq;
26 const u8 *pbmul; /* P multiplier table for B data */
27 const u8 *qmul; /* Q multiplier table (for both) */
28 static const u8 __aligned(16) x0f[16] = {
29 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
30 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
31
32 p = (u8 *)ptrs[disks-2];
33 q = (u8 *)ptrs[disks-1];
34
35 /* Compute syndrome with zero for the missing data pages
36 Use the dead data pages as temporary storage for
37 delta p and delta q */
38 dp = (u8 *)ptrs[faila];
39 ptrs[faila] = (void *)raid6_empty_zero_page;
40 ptrs[disks-2] = dp;
41 dq = (u8 *)ptrs[failb];
42 ptrs[failb] = (void *)raid6_empty_zero_page;
43 ptrs[disks-1] = dq;
44
45 raid6_call.gen_syndrome(disks, bytes, ptrs);
46
47 /* Restore pointer table */
48 ptrs[faila] = dp;
49 ptrs[failb] = dq;
50 ptrs[disks-2] = p;
51 ptrs[disks-1] = q;
52
53 /* Now, pick the proper data tables */
54 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
55 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
56 raid6_gfexp[failb]]];
57
58 kernel_fpu_begin();
59
60 asm volatile("movdqa %0,%%xmm7" : : "m" (x0f[0]));
61
62#ifdef CONFIG_X86_64
63 asm volatile("movdqa %0,%%xmm6" : : "m" (qmul[0]));
64 asm volatile("movdqa %0,%%xmm14" : : "m" (pbmul[0]));
65 asm volatile("movdqa %0,%%xmm15" : : "m" (pbmul[16]));
66#endif
67
68 /* Now do it... */
69 while (bytes) {
70#ifdef CONFIG_X86_64
71 /* xmm6, xmm14, xmm15 */
72
73 asm volatile("movdqa %0,%%xmm1" : : "m" (q[0]));
74 asm volatile("movdqa %0,%%xmm9" : : "m" (q[16]));
75 asm volatile("movdqa %0,%%xmm0" : : "m" (p[0]));
76 asm volatile("movdqa %0,%%xmm8" : : "m" (p[16]));
77 asm volatile("pxor %0,%%xmm1" : : "m" (dq[0]));
78 asm volatile("pxor %0,%%xmm9" : : "m" (dq[16]));
79 asm volatile("pxor %0,%%xmm0" : : "m" (dp[0]));
80 asm volatile("pxor %0,%%xmm8" : : "m" (dp[16]));
81
82 /* xmm0/8 = px */
83
84 asm volatile("movdqa %xmm6,%xmm4");
85 asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
86 asm volatile("movdqa %xmm6,%xmm12");
87 asm volatile("movdqa %xmm5,%xmm13");
88 asm volatile("movdqa %xmm1,%xmm3");
89 asm volatile("movdqa %xmm9,%xmm11");
90 asm volatile("movdqa %xmm0,%xmm2"); /* xmm2/10 = px */
91 asm volatile("movdqa %xmm8,%xmm10");
92 asm volatile("psraw $4,%xmm1");
93 asm volatile("psraw $4,%xmm9");
94 asm volatile("pand %xmm7,%xmm3");
95 asm volatile("pand %xmm7,%xmm11");
96 asm volatile("pand %xmm7,%xmm1");
97 asm volatile("pand %xmm7,%xmm9");
98 asm volatile("pshufb %xmm3,%xmm4");
99 asm volatile("pshufb %xmm11,%xmm12");
100 asm volatile("pshufb %xmm1,%xmm5");
101 asm volatile("pshufb %xmm9,%xmm13");
102 asm volatile("pxor %xmm4,%xmm5");
103 asm volatile("pxor %xmm12,%xmm13");
104
105 /* xmm5/13 = qx */
106
107 asm volatile("movdqa %xmm14,%xmm4");
108 asm volatile("movdqa %xmm15,%xmm1");
109 asm volatile("movdqa %xmm14,%xmm12");
110 asm volatile("movdqa %xmm15,%xmm9");
111 asm volatile("movdqa %xmm2,%xmm3");
112 asm volatile("movdqa %xmm10,%xmm11");
113 asm volatile("psraw $4,%xmm2");
114 asm volatile("psraw $4,%xmm10");
115 asm volatile("pand %xmm7,%xmm3");
116 asm volatile("pand %xmm7,%xmm11");
117 asm volatile("pand %xmm7,%xmm2");
118 asm volatile("pand %xmm7,%xmm10");
119 asm volatile("pshufb %xmm3,%xmm4");
120 asm volatile("pshufb %xmm11,%xmm12");
121 asm volatile("pshufb %xmm2,%xmm1");
122 asm volatile("pshufb %xmm10,%xmm9");
123 asm volatile("pxor %xmm4,%xmm1");
124 asm volatile("pxor %xmm12,%xmm9");
125
126 /* xmm1/9 = pbmul[px] */
127 asm volatile("pxor %xmm5,%xmm1");
128 asm volatile("pxor %xmm13,%xmm9");
129 /* xmm1/9 = db = DQ */
130 asm volatile("movdqa %%xmm1,%0" : "=m" (dq[0]));
131 asm volatile("movdqa %%xmm9,%0" : "=m" (dq[16]));
132
133 asm volatile("pxor %xmm1,%xmm0");
134 asm volatile("pxor %xmm9,%xmm8");
135 asm volatile("movdqa %%xmm0,%0" : "=m" (dp[0]));
136 asm volatile("movdqa %%xmm8,%0" : "=m" (dp[16]));
137
138 bytes -= 32;
139 p += 32;
140 q += 32;
141 dp += 32;
142 dq += 32;
143#else
144 asm volatile("movdqa %0,%%xmm1" : : "m" (*q));
145 asm volatile("movdqa %0,%%xmm0" : : "m" (*p));
146 asm volatile("pxor %0,%%xmm1" : : "m" (*dq));
147 asm volatile("pxor %0,%%xmm0" : : "m" (*dp));
148
149 /* 1 = dq ^ q
150 * 0 = dp ^ p
151 */
152 asm volatile("movdqa %0,%%xmm4" : : "m" (qmul[0]));
153 asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
154
155 asm volatile("movdqa %xmm1,%xmm3");
156 asm volatile("psraw $4,%xmm1");
157 asm volatile("pand %xmm7,%xmm3");
158 asm volatile("pand %xmm7,%xmm1");
159 asm volatile("pshufb %xmm3,%xmm4");
160 asm volatile("pshufb %xmm1,%xmm5");
161 asm volatile("pxor %xmm4,%xmm5");
162
163 asm volatile("movdqa %xmm0,%xmm2"); /* xmm2 = px */
164
165 /* xmm5 = qx */
166
167 asm volatile("movdqa %0,%%xmm4" : : "m" (pbmul[0]));
168 asm volatile("movdqa %0,%%xmm1" : : "m" (pbmul[16]));
169 asm volatile("movdqa %xmm2,%xmm3");
170 asm volatile("psraw $4,%xmm2");
171 asm volatile("pand %xmm7,%xmm3");
172 asm volatile("pand %xmm7,%xmm2");
173 asm volatile("pshufb %xmm3,%xmm4");
174 asm volatile("pshufb %xmm2,%xmm1");
175 asm volatile("pxor %xmm4,%xmm1");
176
177 /* xmm1 = pbmul[px] */
178 asm volatile("pxor %xmm5,%xmm1");
179 /* xmm1 = db = DQ */
180 asm volatile("movdqa %%xmm1,%0" : "=m" (*dq));
181
182 asm volatile("pxor %xmm1,%xmm0");
183 asm volatile("movdqa %%xmm0,%0" : "=m" (*dp));
184
185 bytes -= 16;
186 p += 16;
187 q += 16;
188 dp += 16;
189 dq += 16;
190#endif
191 }
192
193 kernel_fpu_end();
194}
195
196
197void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, void **ptrs)
198{
199 u8 *p, *q, *dq;
200 const u8 *qmul; /* Q multiplier table */
201 static const u8 __aligned(16) x0f[16] = {
202 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
203 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
204
205 p = (u8 *)ptrs[disks-2];
206 q = (u8 *)ptrs[disks-1];
207
208 /* Compute syndrome with zero for the missing data page
209 Use the dead data page as temporary storage for delta q */
210 dq = (u8 *)ptrs[faila];
211 ptrs[faila] = (void *)raid6_empty_zero_page;
212 ptrs[disks-1] = dq;
213
214 raid6_call.gen_syndrome(disks, bytes, ptrs);
215
216 /* Restore pointer table */
217 ptrs[faila] = dq;
218 ptrs[disks-1] = q;
219
220 /* Now, pick the proper data tables */
221 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
222
223 kernel_fpu_begin();
224
225 asm volatile("movdqa %0, %%xmm7" : : "m" (x0f[0]));
226
227 while (bytes) {
228#ifdef CONFIG_X86_64
229 asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
230 asm volatile("movdqa %0, %%xmm4" : : "m" (dq[16]));
231 asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
232 asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
233
234 /* xmm3 = q[0] ^ dq[0] */
235
236 asm volatile("pxor %0, %%xmm4" : : "m" (q[16]));
237 asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
238
239 /* xmm4 = q[16] ^ dq[16] */
240
241 asm volatile("movdqa %xmm3, %xmm6");
242 asm volatile("movdqa %xmm4, %xmm8");
243
244 /* xmm4 = xmm8 = q[16] ^ dq[16] */
245
246 asm volatile("psraw $4, %xmm3");
247 asm volatile("pand %xmm7, %xmm6");
248 asm volatile("pand %xmm7, %xmm3");
249 asm volatile("pshufb %xmm6, %xmm0");
250 asm volatile("pshufb %xmm3, %xmm1");
251 asm volatile("movdqa %0, %%xmm10" : : "m" (qmul[0]));
252 asm volatile("pxor %xmm0, %xmm1");
253 asm volatile("movdqa %0, %%xmm11" : : "m" (qmul[16]));
254
255 /* xmm1 = qmul[q[0] ^ dq[0]] */
256
257 asm volatile("psraw $4, %xmm4");
258 asm volatile("pand %xmm7, %xmm8");
259 asm volatile("pand %xmm7, %xmm4");
260 asm volatile("pshufb %xmm8, %xmm10");
261 asm volatile("pshufb %xmm4, %xmm11");
262 asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
263 asm volatile("pxor %xmm10, %xmm11");
264 asm volatile("movdqa %0, %%xmm12" : : "m" (p[16]));
265
266 /* xmm11 = qmul[q[16] ^ dq[16]] */
267
268 asm volatile("pxor %xmm1, %xmm2");
269
270 /* xmm2 = p[0] ^ qmul[q[0] ^ dq[0]] */
271
272 asm volatile("pxor %xmm11, %xmm12");
273
274 /* xmm12 = p[16] ^ qmul[q[16] ^ dq[16]] */
275
276 asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
277 asm volatile("movdqa %%xmm11, %0" : "=m" (dq[16]));
278
279 asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
280 asm volatile("movdqa %%xmm12, %0" : "=m" (p[16]));
281
282 bytes -= 32;
283 p += 32;
284 q += 32;
285 dq += 32;
286
287#else
288 asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
289 asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
290 asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
291 asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
292
293 /* xmm3 = *q ^ *dq */
294
295 asm volatile("movdqa %xmm3, %xmm6");
296 asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
297 asm volatile("psraw $4, %xmm3");
298 asm volatile("pand %xmm7, %xmm6");
299 asm volatile("pand %xmm7, %xmm3");
300 asm volatile("pshufb %xmm6, %xmm0");
301 asm volatile("pshufb %xmm3, %xmm1");
302 asm volatile("pxor %xmm0, %xmm1");
303
304 /* xmm1 = qmul[*q ^ *dq */
305
306 asm volatile("pxor %xmm1, %xmm2");
307
308 /* xmm2 = *p ^ qmul[*q ^ *dq] */
309
310 asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
311 asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
312
313 bytes -= 16;
314 p += 16;
315 q += 16;
316 dq += 16;
317#endif
318 }
319
320 kernel_fpu_end();
321}
322
323const struct raid6_recov_calls raid6_recov_ssse3 = {
324 .data2 = raid6_2data_recov_ssse3,
325 .datap = raid6_datap_recov_ssse3,
326 .valid = raid6_has_ssse3,
327#ifdef CONFIG_X86_64
328 .name = "ssse3x2",
329#else
330 .name = "ssse3x1",
331#endif
332 .priority = 1,
333};
334
335#endif
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
index aa651697b6dc..c76151d94764 100644
--- a/lib/raid6/test/Makefile
+++ b/lib/raid6/test/Makefile
@@ -23,7 +23,7 @@ RANLIB = ranlib
23all: raid6.a raid6test 23all: raid6.a raid6test
24 24
25raid6.a: int1.o int2.o int4.o int8.o int16.o int32.o mmx.o sse1.o sse2.o \ 25raid6.a: int1.o int2.o int4.o int8.o int16.o int32.o mmx.o sse1.o sse2.o \
26 altivec1.o altivec2.o altivec4.o altivec8.o recov.o algos.o \ 26 altivec1.o altivec2.o altivec4.o altivec8.o recov.o recov_ssse3.o algos.o \
27 tables.o 27 tables.o
28 rm -f $@ 28 rm -f $@
29 $(AR) cq $@ $^ 29 $(AR) cq $@ $^
diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c
index 7a930318b17d..5a485b7a7d3c 100644
--- a/lib/raid6/test/test.c
+++ b/lib/raid6/test/test.c
@@ -90,25 +90,35 @@ static int test_disks(int i, int j)
90int main(int argc, char *argv[]) 90int main(int argc, char *argv[])
91{ 91{
92 const struct raid6_calls *const *algo; 92 const struct raid6_calls *const *algo;
93 const struct raid6_recov_calls *const *ra;
93 int i, j; 94 int i, j;
94 int err = 0; 95 int err = 0;
95 96
96 makedata(); 97 makedata();
97 98
98 for (algo = raid6_algos; *algo; algo++) { 99 for (ra = raid6_recov_algos; *ra; ra++) {
99 if (!(*algo)->valid || (*algo)->valid()) { 100 if ((*ra)->valid && !(*ra)->valid())
100 raid6_call = **algo; 101 continue;
102 raid6_2data_recov = (*ra)->data2;
103 raid6_datap_recov = (*ra)->datap;
101 104
102 /* Nuke syndromes */ 105 printf("using recovery %s\n", (*ra)->name);
103 memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE);
104 106
105 /* Generate assumed good syndrome */ 107 for (algo = raid6_algos; *algo; algo++) {
106 raid6_call.gen_syndrome(NDISKS, PAGE_SIZE, 108 if (!(*algo)->valid || (*algo)->valid()) {
107 (void **)&dataptrs); 109 raid6_call = **algo;
108 110
109 for (i = 0; i < NDISKS-1; i++) 111 /* Nuke syndromes */
110 for (j = i+1; j < NDISKS; j++) 112 memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE);
111 err += test_disks(i, j); 113
114 /* Generate assumed good syndrome */
115 raid6_call.gen_syndrome(NDISKS, PAGE_SIZE,
116 (void **)&dataptrs);
117
118 for (i = 0; i < NDISKS-1; i++)
119 for (j = i+1; j < NDISKS; j++)
120 err += test_disks(i, j);
121 }
112 } 122 }
113 printf("\n"); 123 printf("\n");
114 } 124 }
diff --git a/lib/raid6/x86.h b/lib/raid6/x86.h
index cb2a8c91c886..d55d63232c55 100644
--- a/lib/raid6/x86.h
+++ b/lib/raid6/x86.h
@@ -35,24 +35,29 @@ static inline void kernel_fpu_end(void)
35{ 35{
36} 36}
37 37
38#define __aligned(x) __attribute__((aligned(x)))
39
38#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */ 40#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */
39#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions 41#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions
40 * (fast save and restore) */ 42 * (fast save and restore) */
41#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */ 43#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */
42#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */ 44#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */
45#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */
46#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */
47#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */
43#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ 48#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */
44 49
45/* Should work well enough on modern CPUs for testing */ 50/* Should work well enough on modern CPUs for testing */
46static inline int boot_cpu_has(int flag) 51static inline int boot_cpu_has(int flag)
47{ 52{
48 u32 eax = (flag >> 5) ? 0x80000001 : 1; 53 u32 eax = (flag & 0x20) ? 0x80000001 : 1;
49 u32 edx; 54 u32 ecx, edx;
50 55
51 asm volatile("cpuid" 56 asm volatile("cpuid"
52 : "+a" (eax), "=d" (edx) 57 : "+a" (eax), "=d" (edx), "=c" (ecx)
53 : : "ecx", "ebx"); 58 : : "ebx");
54 59
55 return (edx >> (flag & 31)) & 1; 60 return ((flag & 0x80 ? ecx : edx) >> (flag & 31)) & 1;
56} 61}
57 62
58#endif /* ndef __KERNEL__ */ 63#endif /* ndef __KERNEL__ */
diff --git a/lib/rational.c b/lib/rational.c
index d326da3976f5..f0aa21c2a762 100644
--- a/lib/rational.c
+++ b/lib/rational.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * rational fractions 2 * rational fractions
3 * 3 *
4 * Copyright (C) 2009 emlix GmbH, Oskar Schirmer <os@emlix.com> 4 * Copyright (C) 2009 emlix GmbH, Oskar Schirmer <oskar@scara.com>
5 * 5 *
6 * helper functions when coping with rational numbers 6 * helper functions when coping with rational numbers
7 */ 7 */
diff --git a/lib/spinlock_debug.c b/lib/spinlock_debug.c
index 525d160d44f0..d0ec4f3d1593 100644
--- a/lib/spinlock_debug.c
+++ b/lib/spinlock_debug.c
@@ -58,7 +58,7 @@ static void spin_dump(raw_spinlock_t *lock, const char *msg)
58 printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n", 58 printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
59 msg, raw_smp_processor_id(), 59 msg, raw_smp_processor_id(),
60 current->comm, task_pid_nr(current)); 60 current->comm, task_pid_nr(current));
61 printk(KERN_EMERG " lock: %p, .magic: %08x, .owner: %s/%d, " 61 printk(KERN_EMERG " lock: %ps, .magic: %08x, .owner: %s/%d, "
62 ".owner_cpu: %d\n", 62 ".owner_cpu: %d\n",
63 lock, lock->magic, 63 lock, lock->magic,
64 owner ? owner->comm : "<none>", 64 owner ? owner->comm : "<none>",
diff --git a/lib/stmp_device.c b/lib/stmp_device.c
new file mode 100644
index 000000000000..8ac9bcc4289a
--- /dev/null
+++ b/lib/stmp_device.c
@@ -0,0 +1,80 @@
1/*
2 * Copyright (C) 1999 ARM Limited
3 * Copyright (C) 2000 Deep Blue Solutions Ltd
4 * Copyright 2006-2007,2010 Freescale Semiconductor, Inc. All Rights Reserved.
5 * Copyright 2008 Juergen Beisert, kernel@pengutronix.de
6 * Copyright 2009 Ilya Yanok, Emcraft Systems Ltd, yanok@emcraft.com
7 * Copyright (C) 2011 Wolfram Sang, Pengutronix e.K.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 */
14
15#include <linux/io.h>
16#include <linux/errno.h>
17#include <linux/delay.h>
18#include <linux/module.h>
19#include <linux/stmp_device.h>
20
21#define STMP_MODULE_CLKGATE (1 << 30)
22#define STMP_MODULE_SFTRST (1 << 31)
23
24/*
25 * Clear the bit and poll it cleared. This is usually called with
26 * a reset address and mask being either SFTRST(bit 31) or CLKGATE
27 * (bit 30).
28 */
29static int stmp_clear_poll_bit(void __iomem *addr, u32 mask)
30{
31 int timeout = 0x400;
32
33 writel(mask, addr + STMP_OFFSET_REG_CLR);
34 udelay(1);
35 while ((readl(addr) & mask) && --timeout)
36 /* nothing */;
37
38 return !timeout;
39}
40
41int stmp_reset_block(void __iomem *reset_addr)
42{
43 int ret;
44 int timeout = 0x400;
45
46 /* clear and poll SFTRST */
47 ret = stmp_clear_poll_bit(reset_addr, STMP_MODULE_SFTRST);
48 if (unlikely(ret))
49 goto error;
50
51 /* clear CLKGATE */
52 writel(STMP_MODULE_CLKGATE, reset_addr + STMP_OFFSET_REG_CLR);
53
54 /* set SFTRST to reset the block */
55 writel(STMP_MODULE_SFTRST, reset_addr + STMP_OFFSET_REG_SET);
56 udelay(1);
57
58 /* poll CLKGATE becoming set */
59 while ((!(readl(reset_addr) & STMP_MODULE_CLKGATE)) && --timeout)
60 /* nothing */;
61 if (unlikely(!timeout))
62 goto error;
63
64 /* clear and poll SFTRST */
65 ret = stmp_clear_poll_bit(reset_addr, STMP_MODULE_SFTRST);
66 if (unlikely(ret))
67 goto error;
68
69 /* clear and poll CLKGATE */
70 ret = stmp_clear_poll_bit(reset_addr, STMP_MODULE_CLKGATE);
71 if (unlikely(ret))
72 goto error;
73
74 return 0;
75
76error:
77 pr_err("%s(%p): module reset timeout\n", __func__, reset_addr);
78 return -ETIMEDOUT;
79}
80EXPORT_SYMBOL(stmp_reset_block);
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index dd4ece372699..1cffc223bff5 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -23,15 +23,15 @@
23int string_get_size(u64 size, const enum string_size_units units, 23int string_get_size(u64 size, const enum string_size_units units,
24 char *buf, int len) 24 char *buf, int len)
25{ 25{
26 const char *units_10[] = { "B", "kB", "MB", "GB", "TB", "PB", 26 static const char *units_10[] = { "B", "kB", "MB", "GB", "TB", "PB",
27 "EB", "ZB", "YB", NULL}; 27 "EB", "ZB", "YB", NULL};
28 const char *units_2[] = {"B", "KiB", "MiB", "GiB", "TiB", "PiB", 28 static const char *units_2[] = {"B", "KiB", "MiB", "GiB", "TiB", "PiB",
29 "EiB", "ZiB", "YiB", NULL }; 29 "EiB", "ZiB", "YiB", NULL };
30 const char **units_str[] = { 30 static const char **units_str[] = {
31 [STRING_UNITS_10] = units_10, 31 [STRING_UNITS_10] = units_10,
32 [STRING_UNITS_2] = units_2, 32 [STRING_UNITS_2] = units_2,
33 }; 33 };
34 const unsigned int divisor[] = { 34 static const unsigned int divisor[] = {
35 [STRING_UNITS_10] = 1000, 35 [STRING_UNITS_10] = 1000,
36 [STRING_UNITS_2] = 1024, 36 [STRING_UNITS_2] = 1024,
37 }; 37 };
diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c
new file mode 100644
index 000000000000..bb2b201d6ad0
--- /dev/null
+++ b/lib/strncpy_from_user.c
@@ -0,0 +1,113 @@
1#include <linux/module.h>
2#include <linux/uaccess.h>
3#include <linux/kernel.h>
4#include <linux/errno.h>
5
6#include <asm/byteorder.h>
7#include <asm/word-at-a-time.h>
8
9#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
10#define IS_UNALIGNED(src, dst) 0
11#else
12#define IS_UNALIGNED(src, dst) \
13 (((long) dst | (long) src) & (sizeof(long) - 1))
14#endif
15
16/*
17 * Do a strncpy, return length of string without final '\0'.
18 * 'count' is the user-supplied count (return 'count' if we
19 * hit it), 'max' is the address space maximum (and we return
20 * -EFAULT if we hit it).
21 */
22static inline long do_strncpy_from_user(char *dst, const char __user *src, long count, unsigned long max)
23{
24 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
25 long res = 0;
26
27 /*
28 * Truncate 'max' to the user-specified limit, so that
29 * we only have one limit we need to check in the loop
30 */
31 if (max > count)
32 max = count;
33
34 if (IS_UNALIGNED(src, dst))
35 goto byte_at_a_time;
36
37 while (max >= sizeof(unsigned long)) {
38 unsigned long c, data;
39
40 /* Fall back to byte-at-a-time if we get a page fault */
41 if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
42 break;
43 *(unsigned long *)(dst+res) = c;
44 if (has_zero(c, &data, &constants)) {
45 data = prep_zero_mask(c, data, &constants);
46 data = create_zero_mask(data);
47 return res + find_zero(data);
48 }
49 res += sizeof(unsigned long);
50 max -= sizeof(unsigned long);
51 }
52
53byte_at_a_time:
54 while (max) {
55 char c;
56
57 if (unlikely(__get_user(c,src+res)))
58 return -EFAULT;
59 dst[res] = c;
60 if (!c)
61 return res;
62 res++;
63 max--;
64 }
65
66 /*
67 * Uhhuh. We hit 'max'. But was that the user-specified maximum
68 * too? If so, that's ok - we got as much as the user asked for.
69 */
70 if (res >= count)
71 return res;
72
73 /*
74 * Nope: we hit the address space limit, and we still had more
75 * characters the caller would have wanted. That's an EFAULT.
76 */
77 return -EFAULT;
78}
79
80/**
81 * strncpy_from_user: - Copy a NUL terminated string from userspace.
82 * @dst: Destination address, in kernel space. This buffer must be at
83 * least @count bytes long.
84 * @src: Source address, in user space.
85 * @count: Maximum number of bytes to copy, including the trailing NUL.
86 *
87 * Copies a NUL-terminated string from userspace to kernel space.
88 *
89 * On success, returns the length of the string (not including the trailing
90 * NUL).
91 *
92 * If access to userspace fails, returns -EFAULT (some data may have been
93 * copied).
94 *
95 * If @count is smaller than the length of the string, copies @count bytes
96 * and returns @count.
97 */
98long strncpy_from_user(char *dst, const char __user *src, long count)
99{
100 unsigned long max_addr, src_addr;
101
102 if (unlikely(count <= 0))
103 return 0;
104
105 max_addr = user_addr_max();
106 src_addr = (unsigned long)src;
107 if (likely(src_addr < max_addr)) {
108 unsigned long max = max_addr - src_addr;
109 return do_strncpy_from_user(dst, src, count, max);
110 }
111 return -EFAULT;
112}
113EXPORT_SYMBOL(strncpy_from_user);
diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c
new file mode 100644
index 000000000000..a28df5206d95
--- /dev/null
+++ b/lib/strnlen_user.c
@@ -0,0 +1,138 @@
1#include <linux/kernel.h>
2#include <linux/export.h>
3#include <linux/uaccess.h>
4
5#include <asm/word-at-a-time.h>
6
7/* Set bits in the first 'n' bytes when loaded from memory */
8#ifdef __LITTLE_ENDIAN
9# define aligned_byte_mask(n) ((1ul << 8*(n))-1)
10#else
11# define aligned_byte_mask(n) (~0xfful << (BITS_PER_LONG - 8 - 8*(n)))
12#endif
13
14/*
15 * Do a strnlen, return length of string *with* final '\0'.
16 * 'count' is the user-supplied count, while 'max' is the
17 * address space maximum.
18 *
19 * Return 0 for exceptions (which includes hitting the address
20 * space maximum), or 'count+1' if hitting the user-supplied
21 * maximum count.
22 *
23 * NOTE! We can sometimes overshoot the user-supplied maximum
24 * if it fits in a aligned 'long'. The caller needs to check
25 * the return value against "> max".
26 */
27static inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max)
28{
29 const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
30 long align, res = 0;
31 unsigned long c;
32
33 /*
34 * Truncate 'max' to the user-specified limit, so that
35 * we only have one limit we need to check in the loop
36 */
37 if (max > count)
38 max = count;
39
40 /*
41 * Do everything aligned. But that means that we
42 * need to also expand the maximum..
43 */
44 align = (sizeof(long) - 1) & (unsigned long)src;
45 src -= align;
46 max += align;
47
48 if (unlikely(__get_user(c,(unsigned long __user *)src)))
49 return 0;
50 c |= aligned_byte_mask(align);
51
52 for (;;) {
53 unsigned long data;
54 if (has_zero(c, &data, &constants)) {
55 data = prep_zero_mask(c, data, &constants);
56 data = create_zero_mask(data);
57 return res + find_zero(data) + 1 - align;
58 }
59 res += sizeof(unsigned long);
60 if (unlikely(max < sizeof(unsigned long)))
61 break;
62 max -= sizeof(unsigned long);
63 if (unlikely(__get_user(c,(unsigned long __user *)(src+res))))
64 return 0;
65 }
66 res -= align;
67
68 /*
69 * Uhhuh. We hit 'max'. But was that the user-specified maximum
70 * too? If so, return the marker for "too long".
71 */
72 if (res >= count)
73 return count+1;
74
75 /*
76 * Nope: we hit the address space limit, and we still had more
77 * characters the caller would have wanted. That's 0.
78 */
79 return 0;
80}
81
82/**
83 * strnlen_user: - Get the size of a user string INCLUDING final NUL.
84 * @str: The string to measure.
85 * @count: Maximum count (including NUL character)
86 *
87 * Context: User context only. This function may sleep.
88 *
89 * Get the size of a NUL-terminated string in user space.
90 *
91 * Returns the size of the string INCLUDING the terminating NUL.
92 * If the string is too long, returns 'count+1'.
93 * On exception (or invalid count), returns 0.
94 */
95long strnlen_user(const char __user *str, long count)
96{
97 unsigned long max_addr, src_addr;
98
99 if (unlikely(count <= 0))
100 return 0;
101
102 max_addr = user_addr_max();
103 src_addr = (unsigned long)str;
104 if (likely(src_addr < max_addr)) {
105 unsigned long max = max_addr - src_addr;
106 return do_strnlen_user(str, count, max);
107 }
108 return 0;
109}
110EXPORT_SYMBOL(strnlen_user);
111
112/**
113 * strlen_user: - Get the size of a user string INCLUDING final NUL.
114 * @str: The string to measure.
115 *
116 * Context: User context only. This function may sleep.
117 *
118 * Get the size of a NUL-terminated string in user space.
119 *
120 * Returns the size of the string INCLUDING the terminating NUL.
121 * On exception, returns 0.
122 *
123 * If there is a limit on the length of a valid string, you may wish to
124 * consider using strnlen_user() instead.
125 */
126long strlen_user(const char __user *str)
127{
128 unsigned long max_addr, src_addr;
129
130 max_addr = user_addr_max();
131 src_addr = (unsigned long)str;
132 if (likely(src_addr < max_addr)) {
133 unsigned long max = max_addr - src_addr;
134 return do_strnlen_user(str, ~0ul, max);
135 }
136 return 0;
137}
138EXPORT_SYMBOL(strlen_user);
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 414f46ed1dcd..45bc1f83a5ad 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -130,11 +130,9 @@ void swiotlb_print_info(void)
130 pstart = virt_to_phys(io_tlb_start); 130 pstart = virt_to_phys(io_tlb_start);
131 pend = virt_to_phys(io_tlb_end); 131 pend = virt_to_phys(io_tlb_end);
132 132
133 printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n", 133 printk(KERN_INFO "software IO TLB [mem %#010llx-%#010llx] (%luMB) mapped at [%p-%p]\n",
134 bytes >> 20, io_tlb_start, io_tlb_end); 134 (unsigned long long)pstart, (unsigned long long)pend - 1,
135 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n", 135 bytes >> 20, io_tlb_start, io_tlb_end - 1);
136 (unsigned long long)pstart,
137 (unsigned long long)pend);
138} 136}
139 137
140void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) 138void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c
index d55769d63cb8..bea3f3fa3f02 100644
--- a/lib/test-kstrtox.c
+++ b/lib/test-kstrtox.c
@@ -11,7 +11,7 @@ struct test_fail {
11}; 11};
12 12
13#define DEFINE_TEST_FAIL(test) \ 13#define DEFINE_TEST_FAIL(test) \
14 const struct test_fail test[] __initdata 14 const struct test_fail test[] __initconst
15 15
16#define DECLARE_TEST_OK(type, test_type) \ 16#define DECLARE_TEST_OK(type, test_type) \
17 test_type { \ 17 test_type { \
@@ -21,7 +21,7 @@ struct test_fail {
21 } 21 }
22 22
23#define DEFINE_TEST_OK(type, test) \ 23#define DEFINE_TEST_OK(type, test) \
24 const type test[] __initdata 24 const type test[] __initconst
25 25
26#define TEST_FAIL(fn, type, fmt, test) \ 26#define TEST_FAIL(fn, type, fmt, test) \
27{ \ 27{ \
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index abbabec9720a..c3f36d415bdf 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -112,106 +112,199 @@ int skip_atoi(const char **s)
112/* Decimal conversion is by far the most typical, and is used 112/* Decimal conversion is by far the most typical, and is used
113 * for /proc and /sys data. This directly impacts e.g. top performance 113 * for /proc and /sys data. This directly impacts e.g. top performance
114 * with many processes running. We optimize it for speed 114 * with many processes running. We optimize it for speed
115 * using code from 115 * using ideas described at <http://www.cs.uiowa.edu/~jones/bcd/divide.html>
116 * http://www.cs.uiowa.edu/~jones/bcd/decimal.html 116 * (with permission from the author, Douglas W. Jones).
117 * (with permission from the author, Douglas W. Jones). */ 117 */
118 118
119/* Formats correctly any integer in [0,99999]. 119#if BITS_PER_LONG != 32 || BITS_PER_LONG_LONG != 64
120 * Outputs from one to five digits depending on input. 120/* Formats correctly any integer in [0, 999999999] */
121 * On i386 gcc 4.1.2 -O2: ~250 bytes of code. */
122static noinline_for_stack 121static noinline_for_stack
123char *put_dec_trunc(char *buf, unsigned q) 122char *put_dec_full9(char *buf, unsigned q)
124{ 123{
125 unsigned d3, d2, d1, d0; 124 unsigned r;
126 d1 = (q>>4) & 0xf;
127 d2 = (q>>8) & 0xf;
128 d3 = (q>>12);
129
130 d0 = 6*(d3 + d2 + d1) + (q & 0xf);
131 q = (d0 * 0xcd) >> 11;
132 d0 = d0 - 10*q;
133 *buf++ = d0 + '0'; /* least significant digit */
134 d1 = q + 9*d3 + 5*d2 + d1;
135 if (d1 != 0) {
136 q = (d1 * 0xcd) >> 11;
137 d1 = d1 - 10*q;
138 *buf++ = d1 + '0'; /* next digit */
139
140 d2 = q + 2*d2;
141 if ((d2 != 0) || (d3 != 0)) {
142 q = (d2 * 0xd) >> 7;
143 d2 = d2 - 10*q;
144 *buf++ = d2 + '0'; /* next digit */
145
146 d3 = q + 4*d3;
147 if (d3 != 0) {
148 q = (d3 * 0xcd) >> 11;
149 d3 = d3 - 10*q;
150 *buf++ = d3 + '0'; /* next digit */
151 if (q != 0)
152 *buf++ = q + '0'; /* most sign. digit */
153 }
154 }
155 }
156 125
126 /*
127 * Possible ways to approx. divide by 10
128 * (x * 0x1999999a) >> 32 x < 1073741829 (multiply must be 64-bit)
129 * (x * 0xcccd) >> 19 x < 81920 (x < 262149 when 64-bit mul)
130 * (x * 0x6667) >> 18 x < 43699
131 * (x * 0x3334) >> 17 x < 16389
132 * (x * 0x199a) >> 16 x < 16389
133 * (x * 0x0ccd) >> 15 x < 16389
134 * (x * 0x0667) >> 14 x < 2739
135 * (x * 0x0334) >> 13 x < 1029
136 * (x * 0x019a) >> 12 x < 1029
137 * (x * 0x00cd) >> 11 x < 1029 shorter code than * 0x67 (on i386)
138 * (x * 0x0067) >> 10 x < 179
139 * (x * 0x0034) >> 9 x < 69 same
140 * (x * 0x001a) >> 8 x < 69 same
141 * (x * 0x000d) >> 7 x < 69 same, shortest code (on i386)
142 * (x * 0x0007) >> 6 x < 19
143 * See <http://www.cs.uiowa.edu/~jones/bcd/divide.html>
144 */
145 r = (q * (uint64_t)0x1999999a) >> 32;
146 *buf++ = (q - 10 * r) + '0'; /* 1 */
147 q = (r * (uint64_t)0x1999999a) >> 32;
148 *buf++ = (r - 10 * q) + '0'; /* 2 */
149 r = (q * (uint64_t)0x1999999a) >> 32;
150 *buf++ = (q - 10 * r) + '0'; /* 3 */
151 q = (r * (uint64_t)0x1999999a) >> 32;
152 *buf++ = (r - 10 * q) + '0'; /* 4 */
153 r = (q * (uint64_t)0x1999999a) >> 32;
154 *buf++ = (q - 10 * r) + '0'; /* 5 */
155 /* Now value is under 10000, can avoid 64-bit multiply */
156 q = (r * 0x199a) >> 16;
157 *buf++ = (r - 10 * q) + '0'; /* 6 */
158 r = (q * 0xcd) >> 11;
159 *buf++ = (q - 10 * r) + '0'; /* 7 */
160 q = (r * 0xcd) >> 11;
161 *buf++ = (r - 10 * q) + '0'; /* 8 */
162 *buf++ = q + '0'; /* 9 */
157 return buf; 163 return buf;
158} 164}
159/* Same with if's removed. Always emits five digits */ 165#endif
166
167/* Similar to above but do not pad with zeros.
168 * Code can be easily arranged to print 9 digits too, but our callers
169 * always call put_dec_full9() instead when the number has 9 decimal digits.
170 */
160static noinline_for_stack 171static noinline_for_stack
161char *put_dec_full(char *buf, unsigned q) 172char *put_dec_trunc8(char *buf, unsigned r)
162{ 173{
163 /* BTW, if q is in [0,9999], 8-bit ints will be enough, */ 174 unsigned q;
164 /* but anyway, gcc produces better code with full-sized ints */ 175
165 unsigned d3, d2, d1, d0; 176 /* Copy of previous function's body with added early returns */
166 d1 = (q>>4) & 0xf; 177 q = (r * (uint64_t)0x1999999a) >> 32;
167 d2 = (q>>8) & 0xf; 178 *buf++ = (r - 10 * q) + '0'; /* 2 */
168 d3 = (q>>12); 179 if (q == 0)
180 return buf;
181 r = (q * (uint64_t)0x1999999a) >> 32;
182 *buf++ = (q - 10 * r) + '0'; /* 3 */
183 if (r == 0)
184 return buf;
185 q = (r * (uint64_t)0x1999999a) >> 32;
186 *buf++ = (r - 10 * q) + '0'; /* 4 */
187 if (q == 0)
188 return buf;
189 r = (q * (uint64_t)0x1999999a) >> 32;
190 *buf++ = (q - 10 * r) + '0'; /* 5 */
191 if (r == 0)
192 return buf;
193 q = (r * 0x199a) >> 16;
194 *buf++ = (r - 10 * q) + '0'; /* 6 */
195 if (q == 0)
196 return buf;
197 r = (q * 0xcd) >> 11;
198 *buf++ = (q - 10 * r) + '0'; /* 7 */
199 if (r == 0)
200 return buf;
201 q = (r * 0xcd) >> 11;
202 *buf++ = (r - 10 * q) + '0'; /* 8 */
203 if (q == 0)
204 return buf;
205 *buf++ = q + '0'; /* 9 */
206 return buf;
207}
169 208
170 /* 209/* There are two algorithms to print larger numbers.
171 * Possible ways to approx. divide by 10 210 * One is generic: divide by 1000000000 and repeatedly print
172 * gcc -O2 replaces multiply with shifts and adds 211 * groups of (up to) 9 digits. It's conceptually simple,
173 * (x * 0xcd) >> 11: 11001101 - shorter code than * 0x67 (on i386) 212 * but requires a (unsigned long long) / 1000000000 division.
174 * (x * 0x67) >> 10: 1100111 213 *
175 * (x * 0x34) >> 9: 110100 - same 214 * Second algorithm splits 64-bit unsigned long long into 16-bit chunks,
176 * (x * 0x1a) >> 8: 11010 - same 215 * manipulates them cleverly and generates groups of 4 decimal digits.
177 * (x * 0x0d) >> 7: 1101 - same, shortest code (on i386) 216 * It so happens that it does NOT require long long division.
178 */ 217 *
179 d0 = 6*(d3 + d2 + d1) + (q & 0xf); 218 * If long is > 32 bits, division of 64-bit values is relatively easy,
180 q = (d0 * 0xcd) >> 11; 219 * and we will use the first algorithm.
181 d0 = d0 - 10*q; 220 * If long long is > 64 bits (strange architecture with VERY large long long),
182 *buf++ = d0 + '0'; 221 * second algorithm can't be used, and we again use the first one.
183 d1 = q + 9*d3 + 5*d2 + d1; 222 *
184 q = (d1 * 0xcd) >> 11; 223 * Else (if long is 32 bits and long long is 64 bits) we use second one.
185 d1 = d1 - 10*q; 224 */
186 *buf++ = d1 + '0';
187
188 d2 = q + 2*d2;
189 q = (d2 * 0xd) >> 7;
190 d2 = d2 - 10*q;
191 *buf++ = d2 + '0';
192
193 d3 = q + 4*d3;
194 q = (d3 * 0xcd) >> 11; /* - shorter code */
195 /* q = (d3 * 0x67) >> 10; - would also work */
196 d3 = d3 - 10*q;
197 *buf++ = d3 + '0';
198 *buf++ = q + '0';
199 225
200 return buf; 226#if BITS_PER_LONG != 32 || BITS_PER_LONG_LONG != 64
227
228/* First algorithm: generic */
229
230static
231char *put_dec(char *buf, unsigned long long n)
232{
233 if (n >= 100*1000*1000) {
234 while (n >= 1000*1000*1000)
235 buf = put_dec_full9(buf, do_div(n, 1000*1000*1000));
236 if (n >= 100*1000*1000)
237 return put_dec_full9(buf, n);
238 }
239 return put_dec_trunc8(buf, n);
201} 240}
202/* No inlining helps gcc to use registers better */ 241
242#else
243
244/* Second algorithm: valid only for 64-bit long longs */
245
203static noinline_for_stack 246static noinline_for_stack
204char *put_dec(char *buf, unsigned long long num) 247char *put_dec_full4(char *buf, unsigned q)
205{ 248{
206 while (1) { 249 unsigned r;
207 unsigned rem; 250 r = (q * 0xcccd) >> 19;
208 if (num < 100000) 251 *buf++ = (q - 10 * r) + '0';
209 return put_dec_trunc(buf, num); 252 q = (r * 0x199a) >> 16;
210 rem = do_div(num, 100000); 253 *buf++ = (r - 10 * q) + '0';
211 buf = put_dec_full(buf, rem); 254 r = (q * 0xcd) >> 11;
212 } 255 *buf++ = (q - 10 * r) + '0';
256 *buf++ = r + '0';
257 return buf;
258}
259
260/* Based on code by Douglas W. Jones found at
261 * <http://www.cs.uiowa.edu/~jones/bcd/decimal.html#sixtyfour>
262 * (with permission from the author).
263 * Performs no 64-bit division and hence should be fast on 32-bit machines.
264 */
265static
266char *put_dec(char *buf, unsigned long long n)
267{
268 uint32_t d3, d2, d1, q, h;
269
270 if (n < 100*1000*1000)
271 return put_dec_trunc8(buf, n);
272
273 d1 = ((uint32_t)n >> 16); /* implicit "& 0xffff" */
274 h = (n >> 32);
275 d2 = (h ) & 0xffff;
276 d3 = (h >> 16); /* implicit "& 0xffff" */
277
278 q = 656 * d3 + 7296 * d2 + 5536 * d1 + ((uint32_t)n & 0xffff);
279
280 buf = put_dec_full4(buf, q % 10000);
281 q = q / 10000;
282
283 d1 = q + 7671 * d3 + 9496 * d2 + 6 * d1;
284 buf = put_dec_full4(buf, d1 % 10000);
285 q = d1 / 10000;
286
287 d2 = q + 4749 * d3 + 42 * d2;
288 buf = put_dec_full4(buf, d2 % 10000);
289 q = d2 / 10000;
290
291 d3 = q + 281 * d3;
292 if (!d3)
293 goto done;
294 buf = put_dec_full4(buf, d3 % 10000);
295 q = d3 / 10000;
296 if (!q)
297 goto done;
298 buf = put_dec_full4(buf, q);
299 done:
300 while (buf[-1] == '0')
301 --buf;
302
303 return buf;
213} 304}
214 305
306#endif
307
215/* 308/*
216 * Convert passed number to decimal string. 309 * Convert passed number to decimal string.
217 * Returns the length of string. On buffer overflow, returns 0. 310 * Returns the length of string. On buffer overflow, returns 0.
@@ -220,16 +313,22 @@ char *put_dec(char *buf, unsigned long long num)
220 */ 313 */
221int num_to_str(char *buf, int size, unsigned long long num) 314int num_to_str(char *buf, int size, unsigned long long num)
222{ 315{
223 char tmp[21]; /* Enough for 2^64 in decimal */ 316 char tmp[sizeof(num) * 3];
224 int idx, len; 317 int idx, len;
225 318
226 len = put_dec(tmp, num) - tmp; 319 /* put_dec() may work incorrectly for num = 0 (generate "", not "0") */
320 if (num <= 9) {
321 tmp[0] = '0' + num;
322 len = 1;
323 } else {
324 len = put_dec(tmp, num) - tmp;
325 }
227 326
228 if (len > size) 327 if (len > size)
229 return 0; 328 return 0;
230 for (idx = 0; idx < len; ++idx) 329 for (idx = 0; idx < len; ++idx)
231 buf[idx] = tmp[len - idx - 1]; 330 buf[idx] = tmp[len - idx - 1];
232 return len; 331 return len;
233} 332}
234 333
235#define ZEROPAD 1 /* pad with zero */ 334#define ZEROPAD 1 /* pad with zero */
@@ -284,6 +383,7 @@ char *number(char *buf, char *end, unsigned long long num,
284 char locase; 383 char locase;
285 int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10); 384 int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10);
286 int i; 385 int i;
386 bool is_zero = num == 0LL;
287 387
288 /* locase = 0 or 0x20. ORing digits or letters with 'locase' 388 /* locase = 0 or 0x20. ORing digits or letters with 'locase'
289 * produces same digits or (maybe lowercased) letters */ 389 * produces same digits or (maybe lowercased) letters */
@@ -305,15 +405,16 @@ char *number(char *buf, char *end, unsigned long long num,
305 } 405 }
306 } 406 }
307 if (need_pfx) { 407 if (need_pfx) {
308 spec.field_width--;
309 if (spec.base == 16) 408 if (spec.base == 16)
409 spec.field_width -= 2;
410 else if (!is_zero)
310 spec.field_width--; 411 spec.field_width--;
311 } 412 }
312 413
313 /* generate full string in tmp[], in reverse order */ 414 /* generate full string in tmp[], in reverse order */
314 i = 0; 415 i = 0;
315 if (num == 0) 416 if (num < spec.base)
316 tmp[i++] = '0'; 417 tmp[i++] = digits[num] | locase;
317 /* Generic code, for any base: 418 /* Generic code, for any base:
318 else do { 419 else do {
319 tmp[i++] = (digits[do_div(num,base)] | locase); 420 tmp[i++] = (digits[do_div(num,base)] | locase);
@@ -353,9 +454,11 @@ char *number(char *buf, char *end, unsigned long long num,
353 } 454 }
354 /* "0x" / "0" prefix */ 455 /* "0x" / "0" prefix */
355 if (need_pfx) { 456 if (need_pfx) {
356 if (buf < end) 457 if (spec.base == 16 || !is_zero) {
357 *buf = '0'; 458 if (buf < end)
358 ++buf; 459 *buf = '0';
460 ++buf;
461 }
359 if (spec.base == 16) { 462 if (spec.base == 16) {
360 if (buf < end) 463 if (buf < end)
361 *buf = ('X' | locase); 464 *buf = ('X' | locase);
@@ -436,7 +539,7 @@ char *symbol_string(char *buf, char *end, void *ptr,
436 else if (ext != 'f' && ext != 's') 539 else if (ext != 'f' && ext != 's')
437 sprint_symbol(sym, value); 540 sprint_symbol(sym, value);
438 else 541 else
439 kallsyms_lookup(value, NULL, NULL, NULL, sym); 542 sprint_symbol_no_offset(sym, value);
440 543
441 return string(buf, end, sym, spec); 544 return string(buf, end, sym, spec);
442#else 545#else
@@ -607,7 +710,7 @@ char *ip4_string(char *p, const u8 *addr, const char *fmt)
607 } 710 }
608 for (i = 0; i < 4; i++) { 711 for (i = 0; i < 4; i++) {
609 char temp[3]; /* hold each IP quad in reverse order */ 712 char temp[3]; /* hold each IP quad in reverse order */
610 int digits = put_dec_trunc(temp, addr[index]) - temp; 713 int digits = put_dec_trunc8(temp, addr[index]) - temp;
611 if (leading_zeros) { 714 if (leading_zeros) {
612 if (digits < 3) 715 if (digits < 3)
613 *p++ = '0'; 716 *p++ = '0';
@@ -866,13 +969,15 @@ static noinline_for_stack
866char *pointer(const char *fmt, char *buf, char *end, void *ptr, 969char *pointer(const char *fmt, char *buf, char *end, void *ptr,
867 struct printf_spec spec) 970 struct printf_spec spec)
868{ 971{
972 int default_width = 2 * sizeof(void *) + (spec.flags & SPECIAL ? 2 : 0);
973
869 if (!ptr && *fmt != 'K') { 974 if (!ptr && *fmt != 'K') {
870 /* 975 /*
871 * Print (null) with the same width as a pointer so it makes 976 * Print (null) with the same width as a pointer so it makes
872 * tabular output look nice. 977 * tabular output look nice.
873 */ 978 */
874 if (spec.field_width == -1) 979 if (spec.field_width == -1)
875 spec.field_width = 2 * sizeof(void *); 980 spec.field_width = default_width;
876 return string(buf, end, "(null)", spec); 981 return string(buf, end, "(null)", spec);
877 } 982 }
878 983
@@ -927,7 +1032,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
927 */ 1032 */
928 if (in_irq() || in_serving_softirq() || in_nmi()) { 1033 if (in_irq() || in_serving_softirq() || in_nmi()) {
929 if (spec.field_width == -1) 1034 if (spec.field_width == -1)
930 spec.field_width = 2 * sizeof(void *); 1035 spec.field_width = default_width;
931 return string(buf, end, "pK-error", spec); 1036 return string(buf, end, "pK-error", spec);
932 } 1037 }
933 if (!((kptr_restrict == 0) || 1038 if (!((kptr_restrict == 0) ||
@@ -944,7 +1049,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
944 } 1049 }
945 spec.flags |= SMALL; 1050 spec.flags |= SMALL;
946 if (spec.field_width == -1) { 1051 if (spec.field_width == -1) {
947 spec.field_width = 2 * sizeof(void *); 1052 spec.field_width = default_width;
948 spec.flags |= ZEROPAD; 1053 spec.flags |= ZEROPAD;
949 } 1054 }
950 spec.base = 16; 1055 spec.base = 16;
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-15 21:54:31 -0500