aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig66
-rw-r--r--lib/Kconfig.debug176
-rw-r--r--lib/Makefile20
-rw-r--r--lib/audit.c2
-rw-r--r--lib/average.c61
-rw-r--r--lib/bch.c1368
-rw-r--r--lib/bitmap.c116
-rw-r--r--lib/bsearch.c53
-rw-r--r--lib/btree.c4
-rw-r--r--lib/cpu_rmap.c269
-rw-r--r--lib/debug_locks.c2
-rw-r--r--lib/debugobjects.c11
-rw-r--r--lib/decompress.c5
-rw-r--r--lib/decompress_bunzip2.c8
-rw-r--r--lib/decompress_inflate.c23
-rw-r--r--lib/decompress_unlzma.c85
-rw-r--r--lib/decompress_unlzo.c105
-rw-r--r--lib/decompress_unxz.c397
-rw-r--r--lib/div64.c52
-rw-r--r--lib/dma-debug.c19
-rw-r--r--lib/dynamic_debug.c206
-rw-r--r--lib/find_last_bit.c4
-rw-r--r--lib/find_next_bit.c32
-rw-r--r--lib/flex_array.c97
-rw-r--r--lib/genalloc.c45
-rw-r--r--lib/hexdump.c18
-rw-r--r--lib/idr.c78
-rw-r--r--lib/ioremap.c2
-rw-r--r--lib/kernel_lock.c143
-rw-r--r--lib/kobject.c65
-rw-r--r--lib/kref.c30
-rw-r--r--lib/kstrtox.c250
-rw-r--r--lib/list_debug.c39
-rw-r--r--lib/list_sort.c172
-rw-r--r--lib/locking-selftest.c2
-rw-r--r--lib/lru_cache.c2
-rw-r--r--lib/nlattr.c26
-rw-r--r--lib/parser.c9
-rw-r--r--lib/percpu_counter.c61
-rw-r--r--lib/plist.c135
-rw-r--r--lib/radix-tree.c92
-rw-r--r--lib/rbtree.c3
-rw-r--r--lib/rwsem.c10
-rw-r--r--lib/show_mem.c4
-rw-r--r--lib/string.c29
-rw-r--r--lib/swiotlb.c31
-rw-r--r--lib/test-kstrtox.c739
-rw-r--r--lib/textsearch.c10
-rw-r--r--lib/timerqueue.c107
-rw-r--r--lib/vsprintf.c206
-rw-r--r--lib/xz/Kconfig59
-rw-r--r--lib/xz/Makefile5
-rw-r--r--lib/xz/xz_crc32.c59
-rw-r--r--lib/xz/xz_dec_bcj.c561
-rw-r--r--lib/xz/xz_dec_lzma2.c1171
-rw-r--r--lib/xz/xz_dec_stream.c821
-rw-r--r--lib/xz/xz_dec_syms.c26
-rw-r--r--lib/xz/xz_dec_test.c220
-rw-r--r--lib/xz/xz_lzma2.h204
-rw-r--r--lib/xz/xz_private.h156
-rw-r--r--lib/xz/xz_stream.h62
-rw-r--r--lib/zlib_deflate/deflate.c31
-rw-r--r--lib/zlib_deflate/defutil.h17
63 files changed, 8084 insertions, 797 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index fa9bf2c06199..830181cc7a83 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -19,13 +19,6 @@ config RATIONAL
19config GENERIC_FIND_FIRST_BIT 19config GENERIC_FIND_FIRST_BIT
20 bool 20 bool
21 21
22config GENERIC_FIND_NEXT_BIT
23 bool
24
25config GENERIC_FIND_LAST_BIT
26 bool
27 default y
28
29config CRC_CCITT 22config CRC_CCITT
30 tristate "CRC-CCITT functions" 23 tristate "CRC-CCITT functions"
31 help 24 help
@@ -106,6 +99,8 @@ config LZO_COMPRESS
106config LZO_DECOMPRESS 99config LZO_DECOMPRESS
107 tristate 100 tristate
108 101
102source "lib/xz/Kconfig"
103
109# 104#
110# These all provide a common interface (hence the apparent duplication with 105# These all provide a common interface (hence the apparent duplication with
111# ZLIB_INFLATE; DECOMPRESS_GZIP is just a wrapper.) 106# ZLIB_INFLATE; DECOMPRESS_GZIP is just a wrapper.)
@@ -120,6 +115,10 @@ config DECOMPRESS_BZIP2
120config DECOMPRESS_LZMA 115config DECOMPRESS_LZMA
121 tristate 116 tristate
122 117
118config DECOMPRESS_XZ
119 select XZ_DEC
120 tristate
121
123config DECOMPRESS_LZO 122config DECOMPRESS_LZO
124 select LZO_DECOMPRESS 123 select LZO_DECOMPRESS
125 tristate 124 tristate
@@ -149,6 +148,45 @@ config REED_SOLOMON_DEC16
149 boolean 148 boolean
150 149
151# 150#
151# BCH support is selected if needed
152#
153config BCH
154 tristate
155
156config BCH_CONST_PARAMS
157 boolean
158 help
159 Drivers may select this option to force specific constant
160 values for parameters 'm' (Galois field order) and 't'
161 (error correction capability). Those specific values must
162 be set by declaring default values for symbols BCH_CONST_M
163 and BCH_CONST_T.
164 Doing so will enable extra compiler optimizations,
165 improving encoding and decoding performance up to 2x for
166 usual (m,t) values (typically such that m*t < 200).
167 When this option is selected, the BCH library supports
168 only a single (m,t) configuration. This is mainly useful
169 for NAND flash board drivers requiring known, fixed BCH
170 parameters.
171
172config BCH_CONST_M
173 int
174 range 5 15
175 help
176 Constant value for Galois field order 'm'. If 'k' is the
177 number of data bits to protect, 'm' should be chosen such
178 that (k + m*t) <= 2**m - 1.
179 Drivers should declare a default value for this symbol if
180 they select option BCH_CONST_PARAMS.
181
182config BCH_CONST_T
183 int
184 help
185 Constant value for error correction capability in bits 't'.
186 Drivers should declare a default value for this symbol if
187 they select option BCH_CONST_PARAMS.
188
189#
152# Textsearch support is select'ed if needed 190# Textsearch support is select'ed if needed
153# 191#
154config TEXTSEARCH 192config TEXTSEARCH
@@ -195,6 +233,10 @@ config DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
195 bool "Disable obsolete cpumask functions" if DEBUG_PER_CPU_MAPS 233 bool "Disable obsolete cpumask functions" if DEBUG_PER_CPU_MAPS
196 depends on EXPERIMENTAL && BROKEN 234 depends on EXPERIMENTAL && BROKEN
197 235
236config CPU_RMAP
237 bool
238 depends on SMP
239
198# 240#
199# Netlink attribute parsing support is select'ed if needed 241# Netlink attribute parsing support is select'ed if needed
200# 242#
@@ -210,4 +252,14 @@ config GENERIC_ATOMIC64
210config LRU_CACHE 252config LRU_CACHE
211 tristate 253 tristate
212 254
255config AVERAGE
256 bool "Averaging functions"
257 help
258 This option is provided for the case where no in-kernel-tree
259 modules require averaging functions, but a module built outside
260 the kernel tree does. Such modules that use library averaging
261 functions require Y here.
262
263 If unsure, say N.
264
213endmenu 265endmenu
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 1b4afd2e6ca0..dd373c8ee943 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -7,7 +7,19 @@ config PRINTK_TIME
7 included in printk output. This allows you to measure 7 included in printk output. This allows you to measure
8 the interval between kernel operations, including bootup 8 the interval between kernel operations, including bootup
9 operations. This is useful for identifying long delays 9 operations. This is useful for identifying long delays
10 in kernel startup. 10 in kernel startup. Or add printk.time=1 at boot-time.
11 See Documentation/kernel-parameters.txt
12
13config DEFAULT_MESSAGE_LOGLEVEL
14 int "Default message log level (1-7)"
15 range 1 7
16 default "4"
17 help
18 Default log level for printk statements with no specified priority.
19
20 This was hard-coded to KERN_WARNING since at least 2.6.10 but folks
21 that are auditing their logs closely may want to set it to a lower
22 priority.
11 23
12config ENABLE_WARN_DEPRECATED 24config ENABLE_WARN_DEPRECATED
13 bool "Enable __deprecated logic" 25 bool "Enable __deprecated logic"
@@ -102,11 +114,6 @@ config HEADERS_CHECK
102 114
103config DEBUG_SECTION_MISMATCH 115config DEBUG_SECTION_MISMATCH
104 bool "Enable full Section mismatch analysis" 116 bool "Enable full Section mismatch analysis"
105 depends on UNDEFINED || (BLACKFIN)
106 default y
107 # This option is on purpose disabled for now.
108 # It will be enabled when we are down to a reasonable number
109 # of section mismatch warnings (< 10 for an allyesconfig build)
110 help 117 help
111 The section mismatch analysis checks if there are illegal 118 The section mismatch analysis checks if there are illegal
112 references from one section to another section. 119 references from one section to another section.
@@ -173,7 +180,25 @@ config LOCKUP_DETECTOR
173 An NMI is generated every 60 seconds or so to check for hardlockups. 180 An NMI is generated every 60 seconds or so to check for hardlockups.
174 181
175config HARDLOCKUP_DETECTOR 182config HARDLOCKUP_DETECTOR
176 def_bool LOCKUP_DETECTOR && PERF_EVENTS && HAVE_PERF_EVENTS_NMI 183 def_bool LOCKUP_DETECTOR && PERF_EVENTS && HAVE_PERF_EVENTS_NMI && \
184 !ARCH_HAS_NMI_WATCHDOG
185
186config BOOTPARAM_HARDLOCKUP_PANIC
187 bool "Panic (Reboot) On Hard Lockups"
188 depends on LOCKUP_DETECTOR
189 help
190 Say Y here to enable the kernel to panic on "hard lockups",
191 which are bugs that cause the kernel to loop in kernel
192 mode with interrupts disabled for more than 60 seconds.
193
194 Say N if unsure.
195
196config BOOTPARAM_HARDLOCKUP_PANIC_VALUE
197 int
198 depends on LOCKUP_DETECTOR
199 range 0 1
200 default 0 if !BOOTPARAM_HARDLOCKUP_PANIC
201 default 1 if BOOTPARAM_HARDLOCKUP_PANIC
177 202
178config BOOTPARAM_SOFTLOCKUP_PANIC 203config BOOTPARAM_SOFTLOCKUP_PANIC
179 bool "Panic (Reboot) On Soft Lockups" 204 bool "Panic (Reboot) On Soft Lockups"
@@ -214,6 +239,21 @@ config DETECT_HUNG_TASK
214 enabled then all held locks will also be reported. This 239 enabled then all held locks will also be reported. This
215 feature has negligible overhead. 240 feature has negligible overhead.
216 241
242config DEFAULT_HUNG_TASK_TIMEOUT
243 int "Default timeout for hung task detection (in seconds)"
244 depends on DETECT_HUNG_TASK
245 default 120
246 help
247 This option controls the default timeout (in seconds) used
248 to determine when a task has become non-responsive and should
249 be considered hung.
250
251 It can be adjusted at runtime via the kernel.hung_task_timeout
252 sysctl or by writing a value to /proc/sys/kernel/hung_task_timeout.
253
254 A timeout of 0 disables the check. The default is two minutes.
255 Keeping the default should be fine in most cases.
256
217config BOOTPARAM_HUNG_TASK_PANIC 257config BOOTPARAM_HUNG_TASK_PANIC
218 bool "Panic (Reboot) On Hung Tasks" 258 bool "Panic (Reboot) On Hung Tasks"
219 depends on DETECT_HUNG_TASK 259 depends on DETECT_HUNG_TASK
@@ -313,10 +353,18 @@ config DEBUG_OBJECTS_WORK
313 353
314config DEBUG_OBJECTS_RCU_HEAD 354config DEBUG_OBJECTS_RCU_HEAD
315 bool "Debug RCU callbacks objects" 355 bool "Debug RCU callbacks objects"
316 depends on DEBUG_OBJECTS && PREEMPT 356 depends on DEBUG_OBJECTS
317 help 357 help
318 Enable this to turn on debugging of RCU list heads (call_rcu() usage). 358 Enable this to turn on debugging of RCU list heads (call_rcu() usage).
319 359
360config DEBUG_OBJECTS_PERCPU_COUNTER
361 bool "Debug percpu counter objects"
362 depends on DEBUG_OBJECTS
363 help
364 If you say Y here, additional code will be inserted into the
365 percpu counter routines to track the life time of percpu counter
366 objects and validate the percpu counter operations.
367
320config DEBUG_OBJECTS_ENABLE_DEFAULT 368config DEBUG_OBJECTS_ENABLE_DEFAULT
321 int "debug_objects bootup default value (0-1)" 369 int "debug_objects bootup default value (0-1)"
322 range 0 1 370 range 0 1
@@ -353,7 +401,7 @@ config SLUB_DEBUG_ON
353config SLUB_STATS 401config SLUB_STATS
354 default n 402 default n
355 bool "Enable SLUB performance statistics" 403 bool "Enable SLUB performance statistics"
356 depends on SLUB && SLUB_DEBUG && SYSFS 404 depends on SLUB && SYSFS
357 help 405 help
358 SLUB statistics are useful to debug SLUBs allocation behavior in 406 SLUB statistics are useful to debug SLUBs allocation behavior in
359 order find ways to optimize the allocator. This should never be 407 order find ways to optimize the allocator. This should never be
@@ -366,9 +414,9 @@ config SLUB_STATS
366config DEBUG_KMEMLEAK 414config DEBUG_KMEMLEAK
367 bool "Kernel memory leak detector" 415 bool "Kernel memory leak detector"
368 depends on DEBUG_KERNEL && EXPERIMENTAL && !MEMORY_HOTPLUG && \ 416 depends on DEBUG_KERNEL && EXPERIMENTAL && !MEMORY_HOTPLUG && \
369 (X86 || ARM || PPC || S390 || SPARC64 || SUPERH || MICROBLAZE) 417 (X86 || ARM || PPC || MIPS || S390 || SPARC64 || SUPERH || MICROBLAZE || TILE)
370 418
371 select DEBUG_FS if SYSFS 419 select DEBUG_FS
372 select STACKTRACE if STACKTRACE_SUPPORT 420 select STACKTRACE if STACKTRACE_SUPPORT
373 select KALLSYMS 421 select KALLSYMS
374 select CRC32 422 select CRC32
@@ -402,11 +450,9 @@ config DEBUG_KMEMLEAK_EARLY_LOG_SIZE
402 450
403config DEBUG_KMEMLEAK_TEST 451config DEBUG_KMEMLEAK_TEST
404 tristate "Simple test for the kernel memory leak detector" 452 tristate "Simple test for the kernel memory leak detector"
405 depends on DEBUG_KMEMLEAK 453 depends on DEBUG_KMEMLEAK && m
406 help 454 help
407 Say Y or M here to build a test for the kernel memory leak 455 This option enables a module that explicitly leaks memory.
408 detector. This option enables a module that explicitly leaks
409 memory.
410 456
411 If unsure, say N. 457 If unsure, say N.
412 458
@@ -482,6 +528,7 @@ config PROVE_LOCKING
482 select DEBUG_SPINLOCK 528 select DEBUG_SPINLOCK
483 select DEBUG_MUTEXES 529 select DEBUG_MUTEXES
484 select DEBUG_LOCK_ALLOC 530 select DEBUG_LOCK_ALLOC
531 select TRACE_IRQFLAGS
485 default n 532 default n
486 help 533 help
487 This feature enables the kernel to prove that all locking 534 This feature enables the kernel to prove that all locking
@@ -539,6 +586,23 @@ config PROVE_RCU_REPEATEDLY
539 disabling, allowing multiple RCU-lockdep warnings to be printed 586 disabling, allowing multiple RCU-lockdep warnings to be printed
540 on a single reboot. 587 on a single reboot.
541 588
589 Say Y to allow multiple RCU-lockdep warnings per boot.
590
591 Say N if you are unsure.
592
593config SPARSE_RCU_POINTER
594 bool "RCU debugging: sparse-based checks for pointer usage"
595 default n
596 help
597 This feature enables the __rcu sparse annotation for
598 RCU-protected pointers. This annotation will cause sparse
599 to flag any non-RCU used of annotated pointers. This can be
600 helpful when debugging RCU usage. Please note that this feature
601 is not intended to enforce code cleanliness; it is instead merely
602 a debugging aid.
603
604 Say Y to make sparse flag questionable use of RCU-protected pointers
605
542 Say N if you are unsure. 606 Say N if you are unsure.
543 607
544config LOCKDEP 608config LOCKDEP
@@ -579,11 +643,10 @@ config DEBUG_LOCKDEP
579 of more runtime overhead. 643 of more runtime overhead.
580 644
581config TRACE_IRQFLAGS 645config TRACE_IRQFLAGS
582 depends on DEBUG_KERNEL
583 bool 646 bool
584 default y 647 help
585 depends on TRACE_IRQFLAGS_SUPPORT 648 Enables hooks to interrupt enabling and disabling for
586 depends on PROVE_LOCKING 649 either tracing or lock debugging.
587 650
588config DEBUG_SPINLOCK_SLEEP 651config DEBUG_SPINLOCK_SLEEP
589 bool "Spinlock debugging: sleep-inside-spinlock checking" 652 bool "Spinlock debugging: sleep-inside-spinlock checking"
@@ -607,6 +670,15 @@ config STACKTRACE
607 bool 670 bool
608 depends on STACKTRACE_SUPPORT 671 depends on STACKTRACE_SUPPORT
609 672
673config DEBUG_STACK_USAGE
674 bool "Stack utilization instrumentation"
675 depends on DEBUG_KERNEL
676 help
677 Enables the display of the minimum amount of free stack which each
678 task has ever had available in the sysrq-T and sysrq-P debug output.
679
680 This option will slow down process creation somewhat.
681
610config DEBUG_KOBJECT 682config DEBUG_KOBJECT
611 bool "kobject debugging" 683 bool "kobject debugging"
612 depends on DEBUG_KERNEL 684 depends on DEBUG_KERNEL
@@ -622,10 +694,10 @@ config DEBUG_HIGHMEM
622 Disable for production systems. 694 Disable for production systems.
623 695
624config DEBUG_BUGVERBOSE 696config DEBUG_BUGVERBOSE
625 bool "Verbose BUG() reporting (adds 70K)" if DEBUG_KERNEL && EMBEDDED 697 bool "Verbose BUG() reporting (adds 70K)" if DEBUG_KERNEL && EXPERT
626 depends on BUG 698 depends on BUG
627 depends on ARM || AVR32 || M32R || M68K || SPARC32 || SPARC64 || \ 699 depends on ARM || AVR32 || M32R || M68K || SPARC32 || SPARC64 || \
628 FRV || SUPERH || GENERIC_BUG || BLACKFIN || MN10300 700 FRV || SUPERH || GENERIC_BUG || BLACKFIN || MN10300 || TILE
629 default y 701 default y
630 help 702 help
631 Say Y here to make BUG() panics output the file name and line number 703 Say Y here to make BUG() panics output the file name and line number
@@ -694,8 +766,8 @@ config DEBUG_WRITECOUNT
694 If unsure, say N. 766 If unsure, say N.
695 767
696config DEBUG_MEMORY_INIT 768config DEBUG_MEMORY_INIT
697 bool "Debug memory initialisation" if EMBEDDED 769 bool "Debug memory initialisation" if EXPERT
698 default !EMBEDDED 770 default !EXPERT
699 help 771 help
700 Enable this for additional checks during memory initialisation. 772 Enable this for additional checks during memory initialisation.
701 The sanity checks verify aspects of the VM such as the memory model 773 The sanity checks verify aspects of the VM such as the memory model
@@ -714,6 +786,15 @@ config DEBUG_LIST
714 786
715 If unsure, say N. 787 If unsure, say N.
716 788
789config TEST_LIST_SORT
790 bool "Linked list sorting test"
791 depends on DEBUG_KERNEL
792 help
793 Enable this to turn on 'list_sort()' function test. This test is
794 executed only once during system boot, so affects only boot time.
795
796 If unsure, say N.
797
717config DEBUG_SG 798config DEBUG_SG
718 bool "Debug SG table operations" 799 bool "Debug SG table operations"
719 depends on DEBUG_KERNEL 800 depends on DEBUG_KERNEL
@@ -761,7 +842,7 @@ config ARCH_WANT_FRAME_POINTERS
761config FRAME_POINTER 842config FRAME_POINTER
762 bool "Compile the kernel with frame pointers" 843 bool "Compile the kernel with frame pointers"
763 depends on DEBUG_KERNEL && \ 844 depends on DEBUG_KERNEL && \
764 (CRIS || M68K || M68KNOMMU || FRV || UML || \ 845 (CRIS || M68K || FRV || UML || \
765 AVR32 || SUPERH || BLACKFIN || MN10300) || \ 846 AVR32 || SUPERH || BLACKFIN || MN10300) || \
766 ARCH_WANT_FRAME_POINTERS 847 ARCH_WANT_FRAME_POINTERS
767 default y if (DEBUG_INFO && UML) || ARCH_WANT_FRAME_POINTERS 848 default y if (DEBUG_INFO && UML) || ARCH_WANT_FRAME_POINTERS
@@ -819,22 +900,20 @@ config RCU_TORTURE_TEST_RUNNABLE
819 Say N here if you want the RCU torture tests to start only 900 Say N here if you want the RCU torture tests to start only
820 after being manually enabled via /proc. 901 after being manually enabled via /proc.
821 902
822config RCU_CPU_STALL_DETECTOR 903config RCU_CPU_STALL_TIMEOUT
823 bool "Check for stalled CPUs delaying RCU grace periods" 904 int "RCU CPU stall timeout in seconds"
824 depends on TREE_RCU || TREE_PREEMPT_RCU 905 depends on TREE_RCU || TREE_PREEMPT_RCU
825 default y 906 range 3 300
907 default 60
826 help 908 help
827 This option causes RCU to printk information on which 909 If a given RCU grace period extends more than the specified
828 CPUs are delaying the current grace period, but only when 910 number of seconds, a CPU stall warning is printed. If the
829 the grace period extends for excessive time periods. 911 RCU grace period persists, additional CPU stall warnings are
830 912 printed at more widely spaced intervals.
831 Say N if you want to disable such checks.
832
833 Say Y if you are unsure.
834 913
835config RCU_CPU_STALL_VERBOSE 914config RCU_CPU_STALL_VERBOSE
836 bool "Print additional per-task information for RCU_CPU_STALL_DETECTOR" 915 bool "Print additional per-task information for RCU_CPU_STALL_DETECTOR"
837 depends on RCU_CPU_STALL_DETECTOR && TREE_PREEMPT_RCU 916 depends on TREE_PREEMPT_RCU
838 default y 917 default y
839 help 918 help
840 This option causes RCU to printk detailed per-task information 919 This option causes RCU to printk detailed per-task information
@@ -913,6 +992,17 @@ config DEBUG_FORCE_WEAK_PER_CPU
913 To ensure that generic code follows the above rules, this 992 To ensure that generic code follows the above rules, this
914 option forces all percpu variables to be defined as weak. 993 option forces all percpu variables to be defined as weak.
915 994
995config DEBUG_PER_CPU_MAPS
996 bool "Debug access to per_cpu maps"
997 depends on DEBUG_KERNEL
998 depends on SMP
999 help
1000 Say Y to verify that the per_cpu map being accessed has
1001 been set up. This adds a fair amount of code to kernel memory
1002 and decreases performance.
1003
1004 Say N if unsure.
1005
916config LKDTM 1006config LKDTM
917 tristate "Linux Kernel Dump Test Tool Module" 1007 tristate "Linux Kernel Dump Test Tool Module"
918 depends on DEBUG_FS 1008 depends on DEBUG_FS
@@ -1150,8 +1240,24 @@ config ATOMIC64_SELFTEST
1150 1240
1151 If unsure, say N. 1241 If unsure, say N.
1152 1242
1243config ASYNC_RAID6_TEST
1244 tristate "Self test for hardware accelerated raid6 recovery"
1245 depends on ASYNC_RAID6_RECOV
1246 select ASYNC_MEMCPY
1247 ---help---
1248 This is a one-shot self test that permutes through the
1249 recovery of all the possible two disk failure scenarios for a
1250 N-disk array. Recovery is performed with the asynchronous
1251 raid6 recovery routines, and will optionally use an offload
1252 engine if one is available.
1253
1254 If unsure, say N.
1255
1153source "samples/Kconfig" 1256source "samples/Kconfig"
1154 1257
1155source "lib/Kconfig.kgdb" 1258source "lib/Kconfig.kgdb"
1156 1259
1157source "lib/Kconfig.kmemcheck" 1260source "lib/Kconfig.kmemcheck"
1261
1262config TEST_KSTRTOX
1263 tristate "Test kstrto*() family of functions at runtime"
diff --git a/lib/Makefile b/lib/Makefile
index e6a3763b8212..6b597fdb1898 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -8,11 +8,11 @@ KBUILD_CFLAGS = $(subst -pg,,$(ORIG_CFLAGS))
8endif 8endif
9 9
10lib-y := ctype.o string.o vsprintf.o cmdline.o \ 10lib-y := ctype.o string.o vsprintf.o cmdline.o \
11 rbtree.o radix-tree.o dump_stack.o \ 11 rbtree.o radix-tree.o dump_stack.o timerqueue.o\
12 idr.o int_sqrt.o extable.o prio_tree.o \ 12 idr.o int_sqrt.o extable.o prio_tree.o \
13 sha1.o irq_regs.o reciprocal_div.o argv_split.o \ 13 sha1.o irq_regs.o reciprocal_div.o argv_split.o \
14 proportions.o prio_heap.o ratelimit.o show_mem.o \ 14 proportions.o prio_heap.o ratelimit.o show_mem.o \
15 is_single_threaded.o plist.o decompress.o flex_array.o 15 is_single_threaded.o plist.o decompress.o find_next_bit.o
16 16
17lib-$(CONFIG_MMU) += ioremap.o 17lib-$(CONFIG_MMU) += ioremap.o
18lib-$(CONFIG_SMP) += cpumask.o 18lib-$(CONFIG_SMP) += cpumask.o
@@ -21,7 +21,10 @@ lib-y += kobject.o kref.o klist.o
21 21
22obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \ 22obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \
23 bust_spinlocks.o hexdump.o kasprintf.o bitmap.o scatterlist.o \ 23 bust_spinlocks.o hexdump.o kasprintf.o bitmap.o scatterlist.o \
24 string_helpers.o gcd.o lcm.o list_sort.o uuid.o 24 string_helpers.o gcd.o lcm.o list_sort.o uuid.o flex_array.o \
25 bsearch.o find_last_bit.o
26obj-y += kstrtox.o
27obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
25 28
26ifeq ($(CONFIG_DEBUG_KOBJECT),y) 29ifeq ($(CONFIG_DEBUG_KOBJECT),y)
27CFLAGS_kobject.o += -DDEBUG 30CFLAGS_kobject.o += -DDEBUG
@@ -36,14 +39,10 @@ obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o
36obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o 39obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
37lib-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o 40lib-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o
38lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o 41lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
39lib-$(CONFIG_GENERIC_FIND_FIRST_BIT) += find_next_bit.o
40lib-$(CONFIG_GENERIC_FIND_NEXT_BIT) += find_next_bit.o
41obj-$(CONFIG_GENERIC_FIND_LAST_BIT) += find_last_bit.o
42 42
43CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS)) 43CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS))
44obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o 44obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o
45 45
46obj-$(CONFIG_LOCK_KERNEL) += kernel_lock.o
47obj-$(CONFIG_BTREE) += btree.o 46obj-$(CONFIG_BTREE) += btree.o
48obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o 47obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o
49obj-$(CONFIG_DEBUG_LIST) += list_debug.o 48obj-$(CONFIG_DEBUG_LIST) += list_debug.o
@@ -67,13 +66,16 @@ obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
67obj-$(CONFIG_ZLIB_INFLATE) += zlib_inflate/ 66obj-$(CONFIG_ZLIB_INFLATE) += zlib_inflate/
68obj-$(CONFIG_ZLIB_DEFLATE) += zlib_deflate/ 67obj-$(CONFIG_ZLIB_DEFLATE) += zlib_deflate/
69obj-$(CONFIG_REED_SOLOMON) += reed_solomon/ 68obj-$(CONFIG_REED_SOLOMON) += reed_solomon/
69obj-$(CONFIG_BCH) += bch.o
70obj-$(CONFIG_LZO_COMPRESS) += lzo/ 70obj-$(CONFIG_LZO_COMPRESS) += lzo/
71obj-$(CONFIG_LZO_DECOMPRESS) += lzo/ 71obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
72obj-$(CONFIG_XZ_DEC) += xz/
72obj-$(CONFIG_RAID6_PQ) += raid6/ 73obj-$(CONFIG_RAID6_PQ) += raid6/
73 74
74lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o 75lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
75lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o 76lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
76lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o 77lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o
78lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o
77lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o 79lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
78 80
79obj-$(CONFIG_TEXTSEARCH) += textsearch.o 81obj-$(CONFIG_TEXTSEARCH) += textsearch.o
@@ -106,6 +108,10 @@ obj-$(CONFIG_GENERIC_ATOMIC64) += atomic64.o
106 108
107obj-$(CONFIG_ATOMIC64_SELFTEST) += atomic64_test.o 109obj-$(CONFIG_ATOMIC64_SELFTEST) += atomic64_test.o
108 110
111obj-$(CONFIG_AVERAGE) += average.o
112
113obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o
114
109hostprogs-y := gen_crc32table 115hostprogs-y := gen_crc32table
110clean-files := crc32table.h 116clean-files := crc32table.h
111 117
diff --git a/lib/audit.c b/lib/audit.c
index 8e7dc1c63aa9..76bbed4a20e5 100644
--- a/lib/audit.c
+++ b/lib/audit.c
@@ -36,8 +36,10 @@ int audit_classify_arch(int arch)
36int audit_classify_syscall(int abi, unsigned syscall) 36int audit_classify_syscall(int abi, unsigned syscall)
37{ 37{
38 switch(syscall) { 38 switch(syscall) {
39#ifdef __NR_open
39 case __NR_open: 40 case __NR_open:
40 return 2; 41 return 2;
42#endif
41#ifdef __NR_openat 43#ifdef __NR_openat
42 case __NR_openat: 44 case __NR_openat:
43 return 3; 45 return 3;
diff --git a/lib/average.c b/lib/average.c
new file mode 100644
index 000000000000..5576c2841496
--- /dev/null
+++ b/lib/average.c
@@ -0,0 +1,61 @@
1/*
2 * lib/average.c
3 *
4 * This source code is licensed under the GNU General Public License,
5 * Version 2. See the file COPYING for more details.
6 */
7
8#include <linux/module.h>
9#include <linux/average.h>
10#include <linux/bug.h>
11#include <linux/log2.h>
12
13/**
14 * DOC: Exponentially Weighted Moving Average (EWMA)
15 *
16 * These are generic functions for calculating Exponentially Weighted Moving
17 * Averages (EWMA). We keep a structure with the EWMA parameters and a scaled
18 * up internal representation of the average value to prevent rounding errors.
19 * The factor for scaling up and the exponential weight (or decay rate) have to
20 * be specified thru the init fuction. The structure should not be accessed
21 * directly but only thru the helper functions.
22 */
23
24/**
25 * ewma_init() - Initialize EWMA parameters
26 * @avg: Average structure
27 * @factor: Factor to use for the scaled up internal value. The maximum value
28 * of averages can be ULONG_MAX/(factor*weight). For performance reasons
29 * factor has to be a power of 2.
30 * @weight: Exponential weight, or decay rate. This defines how fast the
31 * influence of older values decreases. For performance reasons weight has
32 * to be a power of 2.
33 *
34 * Initialize the EWMA parameters for a given struct ewma @avg.
35 */
36void ewma_init(struct ewma *avg, unsigned long factor, unsigned long weight)
37{
38 WARN_ON(!is_power_of_2(weight) || !is_power_of_2(factor));
39
40 avg->weight = ilog2(weight);
41 avg->factor = ilog2(factor);
42 avg->internal = 0;
43}
44EXPORT_SYMBOL(ewma_init);
45
46/**
47 * ewma_add() - Exponentially weighted moving average (EWMA)
48 * @avg: Average structure
49 * @val: Current value
50 *
51 * Add a sample to the average.
52 */
53struct ewma *ewma_add(struct ewma *avg, unsigned long val)
54{
55 avg->internal = avg->internal ?
56 (((avg->internal << avg->weight) - avg->internal) +
57 (val << avg->factor)) >> avg->weight :
58 (val << avg->factor);
59 return avg;
60}
61EXPORT_SYMBOL(ewma_add);
diff --git a/lib/bch.c b/lib/bch.c
new file mode 100644
index 000000000000..bc89dfe4d1b3
--- /dev/null
+++ b/lib/bch.c
@@ -0,0 +1,1368 @@
1/*
2 * Generic binary BCH encoding/decoding library
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 as published by
6 * the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 51
15 * Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
16 *
17 * Copyright © 2011 Parrot S.A.
18 *
19 * Author: Ivan Djelic <ivan.djelic@parrot.com>
20 *
21 * Description:
22 *
23 * This library provides runtime configurable encoding/decoding of binary
24 * Bose-Chaudhuri-Hocquenghem (BCH) codes.
25 *
26 * Call init_bch to get a pointer to a newly allocated bch_control structure for
27 * the given m (Galois field order), t (error correction capability) and
28 * (optional) primitive polynomial parameters.
29 *
30 * Call encode_bch to compute and store ecc parity bytes to a given buffer.
31 * Call decode_bch to detect and locate errors in received data.
32 *
33 * On systems supporting hw BCH features, intermediate results may be provided
34 * to decode_bch in order to skip certain steps. See decode_bch() documentation
35 * for details.
36 *
37 * Option CONFIG_BCH_CONST_PARAMS can be used to force fixed values of
38 * parameters m and t; thus allowing extra compiler optimizations and providing
39 * better (up to 2x) encoding performance. Using this option makes sense when
40 * (m,t) are fixed and known in advance, e.g. when using BCH error correction
41 * on a particular NAND flash device.
42 *
43 * Algorithmic details:
44 *
45 * Encoding is performed by processing 32 input bits in parallel, using 4
46 * remainder lookup tables.
47 *
48 * The final stage of decoding involves the following internal steps:
49 * a. Syndrome computation
50 * b. Error locator polynomial computation using Berlekamp-Massey algorithm
51 * c. Error locator root finding (by far the most expensive step)
52 *
53 * In this implementation, step c is not performed using the usual Chien search.
54 * Instead, an alternative approach described in [1] is used. It consists in
55 * factoring the error locator polynomial using the Berlekamp Trace algorithm
56 * (BTA) down to a certain degree (4), after which ad hoc low-degree polynomial
57 * solving techniques [2] are used. The resulting algorithm, called BTZ, yields
58 * much better performance than Chien search for usual (m,t) values (typically
59 * m >= 13, t < 32, see [1]).
60 *
61 * [1] B. Biswas, V. Herbert. Efficient root finding of polynomials over fields
62 * of characteristic 2, in: Western European Workshop on Research in Cryptology
63 * - WEWoRC 2009, Graz, Austria, LNCS, Springer, July 2009, to appear.
64 * [2] [Zin96] V.A. Zinoviev. On the solution of equations of degree 10 over
65 * finite fields GF(2^q). In Rapport de recherche INRIA no 2829, 1996.
66 */
67
68#include <linux/kernel.h>
69#include <linux/errno.h>
70#include <linux/init.h>
71#include <linux/module.h>
72#include <linux/slab.h>
73#include <linux/bitops.h>
74#include <asm/byteorder.h>
75#include <linux/bch.h>
76
77#if defined(CONFIG_BCH_CONST_PARAMS)
78#define GF_M(_p) (CONFIG_BCH_CONST_M)
79#define GF_T(_p) (CONFIG_BCH_CONST_T)
80#define GF_N(_p) ((1 << (CONFIG_BCH_CONST_M))-1)
81#else
82#define GF_M(_p) ((_p)->m)
83#define GF_T(_p) ((_p)->t)
84#define GF_N(_p) ((_p)->n)
85#endif
86
87#define BCH_ECC_WORDS(_p) DIV_ROUND_UP(GF_M(_p)*GF_T(_p), 32)
88#define BCH_ECC_BYTES(_p) DIV_ROUND_UP(GF_M(_p)*GF_T(_p), 8)
89
90#ifndef dbg
91#define dbg(_fmt, args...) do {} while (0)
92#endif
93
94/*
95 * represent a polynomial over GF(2^m)
96 */
97struct gf_poly {
98 unsigned int deg; /* polynomial degree */
99 unsigned int c[0]; /* polynomial terms */
100};
101
102/* given its degree, compute a polynomial size in bytes */
103#define GF_POLY_SZ(_d) (sizeof(struct gf_poly)+((_d)+1)*sizeof(unsigned int))
104
105/* polynomial of degree 1 */
106struct gf_poly_deg1 {
107 struct gf_poly poly;
108 unsigned int c[2];
109};
110
111/*
112 * same as encode_bch(), but process input data one byte at a time
113 */
114static void encode_bch_unaligned(struct bch_control *bch,
115 const unsigned char *data, unsigned int len,
116 uint32_t *ecc)
117{
118 int i;
119 const uint32_t *p;
120 const int l = BCH_ECC_WORDS(bch)-1;
121
122 while (len--) {
123 p = bch->mod8_tab + (l+1)*(((ecc[0] >> 24)^(*data++)) & 0xff);
124
125 for (i = 0; i < l; i++)
126 ecc[i] = ((ecc[i] << 8)|(ecc[i+1] >> 24))^(*p++);
127
128 ecc[l] = (ecc[l] << 8)^(*p);
129 }
130}
131
132/*
133 * convert ecc bytes to aligned, zero-padded 32-bit ecc words
134 */
135static void load_ecc8(struct bch_control *bch, uint32_t *dst,
136 const uint8_t *src)
137{
138 uint8_t pad[4] = {0, 0, 0, 0};
139 unsigned int i, nwords = BCH_ECC_WORDS(bch)-1;
140
141 for (i = 0; i < nwords; i++, src += 4)
142 dst[i] = (src[0] << 24)|(src[1] << 16)|(src[2] << 8)|src[3];
143
144 memcpy(pad, src, BCH_ECC_BYTES(bch)-4*nwords);
145 dst[nwords] = (pad[0] << 24)|(pad[1] << 16)|(pad[2] << 8)|pad[3];
146}
147
148/*
149 * convert 32-bit ecc words to ecc bytes
150 */
151static void store_ecc8(struct bch_control *bch, uint8_t *dst,
152 const uint32_t *src)
153{
154 uint8_t pad[4];
155 unsigned int i, nwords = BCH_ECC_WORDS(bch)-1;
156
157 for (i = 0; i < nwords; i++) {
158 *dst++ = (src[i] >> 24);
159 *dst++ = (src[i] >> 16) & 0xff;
160 *dst++ = (src[i] >> 8) & 0xff;
161 *dst++ = (src[i] >> 0) & 0xff;
162 }
163 pad[0] = (src[nwords] >> 24);
164 pad[1] = (src[nwords] >> 16) & 0xff;
165 pad[2] = (src[nwords] >> 8) & 0xff;
166 pad[3] = (src[nwords] >> 0) & 0xff;
167 memcpy(dst, pad, BCH_ECC_BYTES(bch)-4*nwords);
168}
169
170/**
171 * encode_bch - calculate BCH ecc parity of data
172 * @bch: BCH control structure
173 * @data: data to encode
174 * @len: data length in bytes
175 * @ecc: ecc parity data, must be initialized by caller
176 *
177 * The @ecc parity array is used both as input and output parameter, in order to
178 * allow incremental computations. It should be of the size indicated by member
179 * @ecc_bytes of @bch, and should be initialized to 0 before the first call.
180 *
181 * The exact number of computed ecc parity bits is given by member @ecc_bits of
182 * @bch; it may be less than m*t for large values of t.
183 */
184void encode_bch(struct bch_control *bch, const uint8_t *data,
185 unsigned int len, uint8_t *ecc)
186{
187 const unsigned int l = BCH_ECC_WORDS(bch)-1;
188 unsigned int i, mlen;
189 unsigned long m;
190 uint32_t w, r[l+1];
191 const uint32_t * const tab0 = bch->mod8_tab;
192 const uint32_t * const tab1 = tab0 + 256*(l+1);
193 const uint32_t * const tab2 = tab1 + 256*(l+1);
194 const uint32_t * const tab3 = tab2 + 256*(l+1);
195 const uint32_t *pdata, *p0, *p1, *p2, *p3;
196
197 if (ecc) {
198 /* load ecc parity bytes into internal 32-bit buffer */
199 load_ecc8(bch, bch->ecc_buf, ecc);
200 } else {
201 memset(bch->ecc_buf, 0, sizeof(r));
202 }
203
204 /* process first unaligned data bytes */
205 m = ((unsigned long)data) & 3;
206 if (m) {
207 mlen = (len < (4-m)) ? len : 4-m;
208 encode_bch_unaligned(bch, data, mlen, bch->ecc_buf);
209 data += mlen;
210 len -= mlen;
211 }
212
213 /* process 32-bit aligned data words */
214 pdata = (uint32_t *)data;
215 mlen = len/4;
216 data += 4*mlen;
217 len -= 4*mlen;
218 memcpy(r, bch->ecc_buf, sizeof(r));
219
220 /*
221 * split each 32-bit word into 4 polynomials of weight 8 as follows:
222 *
223 * 31 ...24 23 ...16 15 ... 8 7 ... 0
224 * xxxxxxxx yyyyyyyy zzzzzzzz tttttttt
225 * tttttttt mod g = r0 (precomputed)
226 * zzzzzzzz 00000000 mod g = r1 (precomputed)
227 * yyyyyyyy 00000000 00000000 mod g = r2 (precomputed)
228 * xxxxxxxx 00000000 00000000 00000000 mod g = r3 (precomputed)
229 * xxxxxxxx yyyyyyyy zzzzzzzz tttttttt mod g = r0^r1^r2^r3
230 */
231 while (mlen--) {
232 /* input data is read in big-endian format */
233 w = r[0]^cpu_to_be32(*pdata++);
234 p0 = tab0 + (l+1)*((w >> 0) & 0xff);
235 p1 = tab1 + (l+1)*((w >> 8) & 0xff);
236 p2 = tab2 + (l+1)*((w >> 16) & 0xff);
237 p3 = tab3 + (l+1)*((w >> 24) & 0xff);
238
239 for (i = 0; i < l; i++)
240 r[i] = r[i+1]^p0[i]^p1[i]^p2[i]^p3[i];
241
242 r[l] = p0[l]^p1[l]^p2[l]^p3[l];
243 }
244 memcpy(bch->ecc_buf, r, sizeof(r));
245
246 /* process last unaligned bytes */
247 if (len)
248 encode_bch_unaligned(bch, data, len, bch->ecc_buf);
249
250 /* store ecc parity bytes into original parity buffer */
251 if (ecc)
252 store_ecc8(bch, ecc, bch->ecc_buf);
253}
254EXPORT_SYMBOL_GPL(encode_bch);
255
256static inline int modulo(struct bch_control *bch, unsigned int v)
257{
258 const unsigned int n = GF_N(bch);
259 while (v >= n) {
260 v -= n;
261 v = (v & n) + (v >> GF_M(bch));
262 }
263 return v;
264}
265
266/*
267 * shorter and faster modulo function, only works when v < 2N.
268 */
269static inline int mod_s(struct bch_control *bch, unsigned int v)
270{
271 const unsigned int n = GF_N(bch);
272 return (v < n) ? v : v-n;
273}
274
275static inline int deg(unsigned int poly)
276{
277 /* polynomial degree is the most-significant bit index */
278 return fls(poly)-1;
279}
280
281static inline int parity(unsigned int x)
282{
283 /*
284 * public domain code snippet, lifted from
285 * http://www-graphics.stanford.edu/~seander/bithacks.html
286 */
287 x ^= x >> 1;
288 x ^= x >> 2;
289 x = (x & 0x11111111U) * 0x11111111U;
290 return (x >> 28) & 1;
291}
292
293/* Galois field basic operations: multiply, divide, inverse, etc. */
294
295static inline unsigned int gf_mul(struct bch_control *bch, unsigned int a,
296 unsigned int b)
297{
298 return (a && b) ? bch->a_pow_tab[mod_s(bch, bch->a_log_tab[a]+
299 bch->a_log_tab[b])] : 0;
300}
301
302static inline unsigned int gf_sqr(struct bch_control *bch, unsigned int a)
303{
304 return a ? bch->a_pow_tab[mod_s(bch, 2*bch->a_log_tab[a])] : 0;
305}
306
307static inline unsigned int gf_div(struct bch_control *bch, unsigned int a,
308 unsigned int b)
309{
310 return a ? bch->a_pow_tab[mod_s(bch, bch->a_log_tab[a]+
311 GF_N(bch)-bch->a_log_tab[b])] : 0;
312}
313
314static inline unsigned int gf_inv(struct bch_control *bch, unsigned int a)
315{
316 return bch->a_pow_tab[GF_N(bch)-bch->a_log_tab[a]];
317}
318
319static inline unsigned int a_pow(struct bch_control *bch, int i)
320{
321 return bch->a_pow_tab[modulo(bch, i)];
322}
323
324static inline int a_log(struct bch_control *bch, unsigned int x)
325{
326 return bch->a_log_tab[x];
327}
328
329static inline int a_ilog(struct bch_control *bch, unsigned int x)
330{
331 return mod_s(bch, GF_N(bch)-bch->a_log_tab[x]);
332}
333
334/*
335 * compute 2t syndromes of ecc polynomial, i.e. ecc(a^j) for j=1..2t
336 */
337static void compute_syndromes(struct bch_control *bch, uint32_t *ecc,
338 unsigned int *syn)
339{
340 int i, j, s;
341 unsigned int m;
342 uint32_t poly;
343 const int t = GF_T(bch);
344
345 s = bch->ecc_bits;
346
347 /* make sure extra bits in last ecc word are cleared */
348 m = ((unsigned int)s) & 31;
349 if (m)
350 ecc[s/32] &= ~((1u << (32-m))-1);
351 memset(syn, 0, 2*t*sizeof(*syn));
352
353 /* compute v(a^j) for j=1 .. 2t-1 */
354 do {
355 poly = *ecc++;
356 s -= 32;
357 while (poly) {
358 i = deg(poly);
359 for (j = 0; j < 2*t; j += 2)
360 syn[j] ^= a_pow(bch, (j+1)*(i+s));
361
362 poly ^= (1 << i);
363 }
364 } while (s > 0);
365
366 /* v(a^(2j)) = v(a^j)^2 */
367 for (j = 0; j < t; j++)
368 syn[2*j+1] = gf_sqr(bch, syn[j]);
369}
370
371static void gf_poly_copy(struct gf_poly *dst, struct gf_poly *src)
372{
373 memcpy(dst, src, GF_POLY_SZ(src->deg));
374}
375
376static int compute_error_locator_polynomial(struct bch_control *bch,
377 const unsigned int *syn)
378{
379 const unsigned int t = GF_T(bch);
380 const unsigned int n = GF_N(bch);
381 unsigned int i, j, tmp, l, pd = 1, d = syn[0];
382 struct gf_poly *elp = bch->elp;
383 struct gf_poly *pelp = bch->poly_2t[0];
384 struct gf_poly *elp_copy = bch->poly_2t[1];
385 int k, pp = -1;
386
387 memset(pelp, 0, GF_POLY_SZ(2*t));
388 memset(elp, 0, GF_POLY_SZ(2*t));
389
390 pelp->deg = 0;
391 pelp->c[0] = 1;
392 elp->deg = 0;
393 elp->c[0] = 1;
394
395 /* use simplified binary Berlekamp-Massey algorithm */
396 for (i = 0; (i < t) && (elp->deg <= t); i++) {
397 if (d) {
398 k = 2*i-pp;
399 gf_poly_copy(elp_copy, elp);
400 /* e[i+1](X) = e[i](X)+di*dp^-1*X^2(i-p)*e[p](X) */
401 tmp = a_log(bch, d)+n-a_log(bch, pd);
402 for (j = 0; j <= pelp->deg; j++) {
403 if (pelp->c[j]) {
404 l = a_log(bch, pelp->c[j]);
405 elp->c[j+k] ^= a_pow(bch, tmp+l);
406 }
407 }
408 /* compute l[i+1] = max(l[i]->c[l[p]+2*(i-p]) */
409 tmp = pelp->deg+k;
410 if (tmp > elp->deg) {
411 elp->deg = tmp;
412 gf_poly_copy(pelp, elp_copy);
413 pd = d;
414 pp = 2*i;
415 }
416 }
417 /* di+1 = S(2i+3)+elp[i+1].1*S(2i+2)+...+elp[i+1].lS(2i+3-l) */
418 if (i < t-1) {
419 d = syn[2*i+2];
420 for (j = 1; j <= elp->deg; j++)
421 d ^= gf_mul(bch, elp->c[j], syn[2*i+2-j]);
422 }
423 }
424 dbg("elp=%s\n", gf_poly_str(elp));
425 return (elp->deg > t) ? -1 : (int)elp->deg;
426}
427
428/*
429 * solve a m x m linear system in GF(2) with an expected number of solutions,
430 * and return the number of found solutions
431 */
432static int solve_linear_system(struct bch_control *bch, unsigned int *rows,
433 unsigned int *sol, int nsol)
434{
435 const int m = GF_M(bch);
436 unsigned int tmp, mask;
437 int rem, c, r, p, k, param[m];
438
439 k = 0;
440 mask = 1 << m;
441
442 /* Gaussian elimination */
443 for (c = 0; c < m; c++) {
444 rem = 0;
445 p = c-k;
446 /* find suitable row for elimination */
447 for (r = p; r < m; r++) {
448 if (rows[r] & mask) {
449 if (r != p) {
450 tmp = rows[r];
451 rows[r] = rows[p];
452 rows[p] = tmp;
453 }
454 rem = r+1;
455 break;
456 }
457 }
458 if (rem) {
459 /* perform elimination on remaining rows */
460 tmp = rows[p];
461 for (r = rem; r < m; r++) {
462 if (rows[r] & mask)
463 rows[r] ^= tmp;
464 }
465 } else {
466 /* elimination not needed, store defective row index */
467 param[k++] = c;
468 }
469 mask >>= 1;
470 }
471 /* rewrite system, inserting fake parameter rows */
472 if (k > 0) {
473 p = k;
474 for (r = m-1; r >= 0; r--) {
475 if ((r > m-1-k) && rows[r])
476 /* system has no solution */
477 return 0;
478
479 rows[r] = (p && (r == param[p-1])) ?
480 p--, 1u << (m-r) : rows[r-p];
481 }
482 }
483
484 if (nsol != (1 << k))
485 /* unexpected number of solutions */
486 return 0;
487
488 for (p = 0; p < nsol; p++) {
489 /* set parameters for p-th solution */
490 for (c = 0; c < k; c++)
491 rows[param[c]] = (rows[param[c]] & ~1)|((p >> c) & 1);
492
493 /* compute unique solution */
494 tmp = 0;
495 for (r = m-1; r >= 0; r--) {
496 mask = rows[r] & (tmp|1);
497 tmp |= parity(mask) << (m-r);
498 }
499 sol[p] = tmp >> 1;
500 }
501 return nsol;
502}
503
504/*
505 * this function builds and solves a linear system for finding roots of a degree
506 * 4 affine monic polynomial X^4+aX^2+bX+c over GF(2^m).
507 */
508static int find_affine4_roots(struct bch_control *bch, unsigned int a,
509 unsigned int b, unsigned int c,
510 unsigned int *roots)
511{
512 int i, j, k;
513 const int m = GF_M(bch);
514 unsigned int mask = 0xff, t, rows[16] = {0,};
515
516 j = a_log(bch, b);
517 k = a_log(bch, a);
518 rows[0] = c;
519
520 /* buid linear system to solve X^4+aX^2+bX+c = 0 */
521 for (i = 0; i < m; i++) {
522 rows[i+1] = bch->a_pow_tab[4*i]^
523 (a ? bch->a_pow_tab[mod_s(bch, k)] : 0)^
524 (b ? bch->a_pow_tab[mod_s(bch, j)] : 0);
525 j++;
526 k += 2;
527 }
528 /*
529 * transpose 16x16 matrix before passing it to linear solver
530 * warning: this code assumes m < 16
531 */
532 for (j = 8; j != 0; j >>= 1, mask ^= (mask << j)) {
533 for (k = 0; k < 16; k = (k+j+1) & ~j) {
534 t = ((rows[k] >> j)^rows[k+j]) & mask;
535 rows[k] ^= (t << j);
536 rows[k+j] ^= t;
537 }
538 }
539 return solve_linear_system(bch, rows, roots, 4);
540}
541
542/*
543 * compute root r of a degree 1 polynomial over GF(2^m) (returned as log(1/r))
544 */
545static int find_poly_deg1_roots(struct bch_control *bch, struct gf_poly *poly,
546 unsigned int *roots)
547{
548 int n = 0;
549
550 if (poly->c[0])
551 /* poly[X] = bX+c with c!=0, root=c/b */
552 roots[n++] = mod_s(bch, GF_N(bch)-bch->a_log_tab[poly->c[0]]+
553 bch->a_log_tab[poly->c[1]]);
554 return n;
555}
556
557/*
558 * compute roots of a degree 2 polynomial over GF(2^m)
559 */
560static int find_poly_deg2_roots(struct bch_control *bch, struct gf_poly *poly,
561 unsigned int *roots)
562{
563 int n = 0, i, l0, l1, l2;
564 unsigned int u, v, r;
565
566 if (poly->c[0] && poly->c[1]) {
567
568 l0 = bch->a_log_tab[poly->c[0]];
569 l1 = bch->a_log_tab[poly->c[1]];
570 l2 = bch->a_log_tab[poly->c[2]];
571
572 /* using z=a/bX, transform aX^2+bX+c into z^2+z+u (u=ac/b^2) */
573 u = a_pow(bch, l0+l2+2*(GF_N(bch)-l1));
574 /*
575 * let u = sum(li.a^i) i=0..m-1; then compute r = sum(li.xi):
576 * r^2+r = sum(li.(xi^2+xi)) = sum(li.(a^i+Tr(a^i).a^k)) =
577 * u + sum(li.Tr(a^i).a^k) = u+a^k.Tr(sum(li.a^i)) = u+a^k.Tr(u)
578 * i.e. r and r+1 are roots iff Tr(u)=0
579 */
580 r = 0;
581 v = u;
582 while (v) {
583 i = deg(v);
584 r ^= bch->xi_tab[i];
585 v ^= (1 << i);
586 }
587 /* verify root */
588 if ((gf_sqr(bch, r)^r) == u) {
589 /* reverse z=a/bX transformation and compute log(1/r) */
590 roots[n++] = modulo(bch, 2*GF_N(bch)-l1-
591 bch->a_log_tab[r]+l2);
592 roots[n++] = modulo(bch, 2*GF_N(bch)-l1-
593 bch->a_log_tab[r^1]+l2);
594 }
595 }
596 return n;
597}
598
599/*
600 * compute roots of a degree 3 polynomial over GF(2^m)
601 */
602static int find_poly_deg3_roots(struct bch_control *bch, struct gf_poly *poly,
603 unsigned int *roots)
604{
605 int i, n = 0;
606 unsigned int a, b, c, a2, b2, c2, e3, tmp[4];
607
608 if (poly->c[0]) {
609 /* transform polynomial into monic X^3 + a2X^2 + b2X + c2 */
610 e3 = poly->c[3];
611 c2 = gf_div(bch, poly->c[0], e3);
612 b2 = gf_div(bch, poly->c[1], e3);
613 a2 = gf_div(bch, poly->c[2], e3);
614
615 /* (X+a2)(X^3+a2X^2+b2X+c2) = X^4+aX^2+bX+c (affine) */
616 c = gf_mul(bch, a2, c2); /* c = a2c2 */
617 b = gf_mul(bch, a2, b2)^c2; /* b = a2b2 + c2 */
618 a = gf_sqr(bch, a2)^b2; /* a = a2^2 + b2 */
619
620 /* find the 4 roots of this affine polynomial */
621 if (find_affine4_roots(bch, a, b, c, tmp) == 4) {
622 /* remove a2 from final list of roots */
623 for (i = 0; i < 4; i++) {
624 if (tmp[i] != a2)
625 roots[n++] = a_ilog(bch, tmp[i]);
626 }
627 }
628 }
629 return n;
630}
631
632/*
633 * compute roots of a degree 4 polynomial over GF(2^m)
634 */
635static int find_poly_deg4_roots(struct bch_control *bch, struct gf_poly *poly,
636 unsigned int *roots)
637{
638 int i, l, n = 0;
639 unsigned int a, b, c, d, e = 0, f, a2, b2, c2, e4;
640
641 if (poly->c[0] == 0)
642 return 0;
643
644 /* transform polynomial into monic X^4 + aX^3 + bX^2 + cX + d */
645 e4 = poly->c[4];
646 d = gf_div(bch, poly->c[0], e4);
647 c = gf_div(bch, poly->c[1], e4);
648 b = gf_div(bch, poly->c[2], e4);
649 a = gf_div(bch, poly->c[3], e4);
650
651 /* use Y=1/X transformation to get an affine polynomial */
652 if (a) {
653 /* first, eliminate cX by using z=X+e with ae^2+c=0 */
654 if (c) {
655 /* compute e such that e^2 = c/a */
656 f = gf_div(bch, c, a);
657 l = a_log(bch, f);
658 l += (l & 1) ? GF_N(bch) : 0;
659 e = a_pow(bch, l/2);
660 /*
661 * use transformation z=X+e:
662 * z^4+e^4 + a(z^3+ez^2+e^2z+e^3) + b(z^2+e^2) +cz+ce+d
663 * z^4 + az^3 + (ae+b)z^2 + (ae^2+c)z+e^4+be^2+ae^3+ce+d
664 * z^4 + az^3 + (ae+b)z^2 + e^4+be^2+d
665 * z^4 + az^3 + b'z^2 + d'
666 */
667 d = a_pow(bch, 2*l)^gf_mul(bch, b, f)^d;
668 b = gf_mul(bch, a, e)^b;
669 }
670 /* now, use Y=1/X to get Y^4 + b/dY^2 + a/dY + 1/d */
671 if (d == 0)
672 /* assume all roots have multiplicity 1 */
673 return 0;
674
675 c2 = gf_inv(bch, d);
676 b2 = gf_div(bch, a, d);
677 a2 = gf_div(bch, b, d);
678 } else {
679 /* polynomial is already affine */
680 c2 = d;
681 b2 = c;
682 a2 = b;
683 }
684 /* find the 4 roots of this affine polynomial */
685 if (find_affine4_roots(bch, a2, b2, c2, roots) == 4) {
686 for (i = 0; i < 4; i++) {
687 /* post-process roots (reverse transformations) */
688 f = a ? gf_inv(bch, roots[i]) : roots[i];
689 roots[i] = a_ilog(bch, f^e);
690 }
691 n = 4;
692 }
693 return n;
694}
695
696/*
697 * build monic, log-based representation of a polynomial
698 */
699static void gf_poly_logrep(struct bch_control *bch,
700 const struct gf_poly *a, int *rep)
701{
702 int i, d = a->deg, l = GF_N(bch)-a_log(bch, a->c[a->deg]);
703
704 /* represent 0 values with -1; warning, rep[d] is not set to 1 */
705 for (i = 0; i < d; i++)
706 rep[i] = a->c[i] ? mod_s(bch, a_log(bch, a->c[i])+l) : -1;
707}
708
709/*
710 * compute polynomial Euclidean division remainder in GF(2^m)[X]
711 */
712static void gf_poly_mod(struct bch_control *bch, struct gf_poly *a,
713 const struct gf_poly *b, int *rep)
714{
715 int la, p, m;
716 unsigned int i, j, *c = a->c;
717 const unsigned int d = b->deg;
718
719 if (a->deg < d)
720 return;
721
722 /* reuse or compute log representation of denominator */
723 if (!rep) {
724 rep = bch->cache;
725 gf_poly_logrep(bch, b, rep);
726 }
727
728 for (j = a->deg; j >= d; j--) {
729 if (c[j]) {
730 la = a_log(bch, c[j]);
731 p = j-d;
732 for (i = 0; i < d; i++, p++) {
733 m = rep[i];
734 if (m >= 0)
735 c[p] ^= bch->a_pow_tab[mod_s(bch,
736 m+la)];
737 }
738 }
739 }
740 a->deg = d-1;
741 while (!c[a->deg] && a->deg)
742 a->deg--;
743}
744
745/*
746 * compute polynomial Euclidean division quotient in GF(2^m)[X]
747 */
748static void gf_poly_div(struct bch_control *bch, struct gf_poly *a,
749 const struct gf_poly *b, struct gf_poly *q)
750{
751 if (a->deg >= b->deg) {
752 q->deg = a->deg-b->deg;
753 /* compute a mod b (modifies a) */
754 gf_poly_mod(bch, a, b, NULL);
755 /* quotient is stored in upper part of polynomial a */
756 memcpy(q->c, &a->c[b->deg], (1+q->deg)*sizeof(unsigned int));
757 } else {
758 q->deg = 0;
759 q->c[0] = 0;
760 }
761}
762
763/*
764 * compute polynomial GCD (Greatest Common Divisor) in GF(2^m)[X]
765 */
766static struct gf_poly *gf_poly_gcd(struct bch_control *bch, struct gf_poly *a,
767 struct gf_poly *b)
768{
769 struct gf_poly *tmp;
770
771 dbg("gcd(%s,%s)=", gf_poly_str(a), gf_poly_str(b));
772
773 if (a->deg < b->deg) {
774 tmp = b;
775 b = a;
776 a = tmp;
777 }
778
779 while (b->deg > 0) {
780 gf_poly_mod(bch, a, b, NULL);
781 tmp = b;
782 b = a;
783 a = tmp;
784 }
785
786 dbg("%s\n", gf_poly_str(a));
787
788 return a;
789}
790
791/*
792 * Given a polynomial f and an integer k, compute Tr(a^kX) mod f
793 * This is used in Berlekamp Trace algorithm for splitting polynomials
794 */
795static void compute_trace_bk_mod(struct bch_control *bch, int k,
796 const struct gf_poly *f, struct gf_poly *z,
797 struct gf_poly *out)
798{
799 const int m = GF_M(bch);
800 int i, j;
801
802 /* z contains z^2j mod f */
803 z->deg = 1;
804 z->c[0] = 0;
805 z->c[1] = bch->a_pow_tab[k];
806
807 out->deg = 0;
808 memset(out, 0, GF_POLY_SZ(f->deg));
809
810 /* compute f log representation only once */
811 gf_poly_logrep(bch, f, bch->cache);
812
813 for (i = 0; i < m; i++) {
814 /* add a^(k*2^i)(z^(2^i) mod f) and compute (z^(2^i) mod f)^2 */
815 for (j = z->deg; j >= 0; j--) {
816 out->c[j] ^= z->c[j];
817 z->c[2*j] = gf_sqr(bch, z->c[j]);
818 z->c[2*j+1] = 0;
819 }
820 if (z->deg > out->deg)
821 out->deg = z->deg;
822
823 if (i < m-1) {
824 z->deg *= 2;
825 /* z^(2(i+1)) mod f = (z^(2^i) mod f)^2 mod f */
826 gf_poly_mod(bch, z, f, bch->cache);
827 }
828 }
829 while (!out->c[out->deg] && out->deg)
830 out->deg--;
831
832 dbg("Tr(a^%d.X) mod f = %s\n", k, gf_poly_str(out));
833}
834
835/*
836 * factor a polynomial using Berlekamp Trace algorithm (BTA)
837 */
838static void factor_polynomial(struct bch_control *bch, int k, struct gf_poly *f,
839 struct gf_poly **g, struct gf_poly **h)
840{
841 struct gf_poly *f2 = bch->poly_2t[0];
842 struct gf_poly *q = bch->poly_2t[1];
843 struct gf_poly *tk = bch->poly_2t[2];
844 struct gf_poly *z = bch->poly_2t[3];
845 struct gf_poly *gcd;
846
847 dbg("factoring %s...\n", gf_poly_str(f));
848
849 *g = f;
850 *h = NULL;
851
852 /* tk = Tr(a^k.X) mod f */
853 compute_trace_bk_mod(bch, k, f, z, tk);
854
855 if (tk->deg > 0) {
856 /* compute g = gcd(f, tk) (destructive operation) */
857 gf_poly_copy(f2, f);
858 gcd = gf_poly_gcd(bch, f2, tk);
859 if (gcd->deg < f->deg) {
860 /* compute h=f/gcd(f,tk); this will modify f and q */
861 gf_poly_div(bch, f, gcd, q);
862 /* store g and h in-place (clobbering f) */
863 *h = &((struct gf_poly_deg1 *)f)[gcd->deg].poly;
864 gf_poly_copy(*g, gcd);
865 gf_poly_copy(*h, q);
866 }
867 }
868}
869
870/*
871 * find roots of a polynomial, using BTZ algorithm; see the beginning of this
872 * file for details
873 */
874static int find_poly_roots(struct bch_control *bch, unsigned int k,
875 struct gf_poly *poly, unsigned int *roots)
876{
877 int cnt;
878 struct gf_poly *f1, *f2;
879
880 switch (poly->deg) {
881 /* handle low degree polynomials with ad hoc techniques */
882 case 1:
883 cnt = find_poly_deg1_roots(bch, poly, roots);
884 break;
885 case 2:
886 cnt = find_poly_deg2_roots(bch, poly, roots);
887 break;
888 case 3:
889 cnt = find_poly_deg3_roots(bch, poly, roots);
890 break;
891 case 4:
892 cnt = find_poly_deg4_roots(bch, poly, roots);
893 break;
894 default:
895 /* factor polynomial using Berlekamp Trace Algorithm (BTA) */
896 cnt = 0;
897 if (poly->deg && (k <= GF_M(bch))) {
898 factor_polynomial(bch, k, poly, &f1, &f2);
899 if (f1)
900 cnt += find_poly_roots(bch, k+1, f1, roots);
901 if (f2)
902 cnt += find_poly_roots(bch, k+1, f2, roots+cnt);
903 }
904 break;
905 }
906 return cnt;
907}
908
909#if defined(USE_CHIEN_SEARCH)
910/*
911 * exhaustive root search (Chien) implementation - not used, included only for
912 * reference/comparison tests
913 */
914static int chien_search(struct bch_control *bch, unsigned int len,
915 struct gf_poly *p, unsigned int *roots)
916{
917 int m;
918 unsigned int i, j, syn, syn0, count = 0;
919 const unsigned int k = 8*len+bch->ecc_bits;
920
921 /* use a log-based representation of polynomial */
922 gf_poly_logrep(bch, p, bch->cache);
923 bch->cache[p->deg] = 0;
924 syn0 = gf_div(bch, p->c[0], p->c[p->deg]);
925
926 for (i = GF_N(bch)-k+1; i <= GF_N(bch); i++) {
927 /* compute elp(a^i) */
928 for (j = 1, syn = syn0; j <= p->deg; j++) {
929 m = bch->cache[j];
930 if (m >= 0)
931 syn ^= a_pow(bch, m+j*i);
932 }
933 if (syn == 0) {
934 roots[count++] = GF_N(bch)-i;
935 if (count == p->deg)
936 break;
937 }
938 }
939 return (count == p->deg) ? count : 0;
940}
941#define find_poly_roots(_p, _k, _elp, _loc) chien_search(_p, len, _elp, _loc)
942#endif /* USE_CHIEN_SEARCH */
943
944/**
945 * decode_bch - decode received codeword and find bit error locations
946 * @bch: BCH control structure
947 * @data: received data, ignored if @calc_ecc is provided
948 * @len: data length in bytes, must always be provided
949 * @recv_ecc: received ecc, if NULL then assume it was XORed in @calc_ecc
950 * @calc_ecc: calculated ecc, if NULL then calc_ecc is computed from @data
951 * @syn: hw computed syndrome data (if NULL, syndrome is calculated)
952 * @errloc: output array of error locations
953 *
954 * Returns:
955 * The number of errors found, or -EBADMSG if decoding failed, or -EINVAL if
956 * invalid parameters were provided
957 *
958 * Depending on the available hw BCH support and the need to compute @calc_ecc
959 * separately (using encode_bch()), this function should be called with one of
960 * the following parameter configurations -
961 *
962 * by providing @data and @recv_ecc only:
963 * decode_bch(@bch, @data, @len, @recv_ecc, NULL, NULL, @errloc)
964 *
965 * by providing @recv_ecc and @calc_ecc:
966 * decode_bch(@bch, NULL, @len, @recv_ecc, @calc_ecc, NULL, @errloc)
967 *
968 * by providing ecc = recv_ecc XOR calc_ecc:
969 * decode_bch(@bch, NULL, @len, NULL, ecc, NULL, @errloc)
970 *
971 * by providing syndrome results @syn:
972 * decode_bch(@bch, NULL, @len, NULL, NULL, @syn, @errloc)
973 *
974 * Once decode_bch() has successfully returned with a positive value, error
975 * locations returned in array @errloc should be interpreted as follows -
976 *
977 * if (errloc[n] >= 8*len), then n-th error is located in ecc (no need for
978 * data correction)
979 *
980 * if (errloc[n] < 8*len), then n-th error is located in data and can be
981 * corrected with statement data[errloc[n]/8] ^= 1 << (errloc[n] % 8);
982 *
983 * Note that this function does not perform any data correction by itself, it
984 * merely indicates error locations.
985 */
986int decode_bch(struct bch_control *bch, const uint8_t *data, unsigned int len,
987 const uint8_t *recv_ecc, const uint8_t *calc_ecc,
988 const unsigned int *syn, unsigned int *errloc)
989{
990 const unsigned int ecc_words = BCH_ECC_WORDS(bch);
991 unsigned int nbits;
992 int i, err, nroots;
993 uint32_t sum;
994
995 /* sanity check: make sure data length can be handled */
996 if (8*len > (bch->n-bch->ecc_bits))
997 return -EINVAL;
998
999 /* if caller does not provide syndromes, compute them */
1000 if (!syn) {
1001 if (!calc_ecc) {
1002 /* compute received data ecc into an internal buffer */
1003 if (!data || !recv_ecc)
1004 return -EINVAL;
1005 encode_bch(bch, data, len, NULL);
1006 } else {
1007 /* load provided calculated ecc */
1008 load_ecc8(bch, bch->ecc_buf, calc_ecc);
1009 }
1010 /* load received ecc or assume it was XORed in calc_ecc */
1011 if (recv_ecc) {
1012 load_ecc8(bch, bch->ecc_buf2, recv_ecc);
1013 /* XOR received and calculated ecc */
1014 for (i = 0, sum = 0; i < (int)ecc_words; i++) {
1015 bch->ecc_buf[i] ^= bch->ecc_buf2[i];
1016 sum |= bch->ecc_buf[i];
1017 }
1018 if (!sum)
1019 /* no error found */
1020 return 0;
1021 }
1022 compute_syndromes(bch, bch->ecc_buf, bch->syn);
1023 syn = bch->syn;
1024 }
1025
1026 err = compute_error_locator_polynomial(bch, syn);
1027 if (err > 0) {
1028 nroots = find_poly_roots(bch, 1, bch->elp, errloc);
1029 if (err != nroots)
1030 err = -1;
1031 }
1032 if (err > 0) {
1033 /* post-process raw error locations for easier correction */
1034 nbits = (len*8)+bch->ecc_bits;
1035 for (i = 0; i < err; i++) {
1036 if (errloc[i] >= nbits) {
1037 err = -1;
1038 break;
1039 }
1040 errloc[i] = nbits-1-errloc[i];
1041 errloc[i] = (errloc[i] & ~7)|(7-(errloc[i] & 7));
1042 }
1043 }
1044 return (err >= 0) ? err : -EBADMSG;
1045}
1046EXPORT_SYMBOL_GPL(decode_bch);
1047
1048/*
1049 * generate Galois field lookup tables
1050 */
1051static int build_gf_tables(struct bch_control *bch, unsigned int poly)
1052{
1053 unsigned int i, x = 1;
1054 const unsigned int k = 1 << deg(poly);
1055
1056 /* primitive polynomial must be of degree m */
1057 if (k != (1u << GF_M(bch)))
1058 return -1;
1059
1060 for (i = 0; i < GF_N(bch); i++) {
1061 bch->a_pow_tab[i] = x;
1062 bch->a_log_tab[x] = i;
1063 if (i && (x == 1))
1064 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
1065 return -1;
1066 x <<= 1;
1067 if (x & k)
1068 x ^= poly;
1069 }
1070 bch->a_pow_tab[GF_N(bch)] = 1;
1071 bch->a_log_tab[0] = 0;
1072
1073 return 0;
1074}
1075
1076/*
1077 * compute generator polynomial remainder tables for fast encoding
1078 */
1079static void build_mod8_tables(struct bch_control *bch, const uint32_t *g)
1080{
1081 int i, j, b, d;
1082 uint32_t data, hi, lo, *tab;
1083 const int l = BCH_ECC_WORDS(bch);
1084 const int plen = DIV_ROUND_UP(bch->ecc_bits+1, 32);
1085 const int ecclen = DIV_ROUND_UP(bch->ecc_bits, 32);
1086
1087 memset(bch->mod8_tab, 0, 4*256*l*sizeof(*bch->mod8_tab));
1088
1089 for (i = 0; i < 256; i++) {
1090 /* p(X)=i is a small polynomial of weight <= 8 */
1091 for (b = 0; b < 4; b++) {
1092 /* we want to compute (p(X).X^(8*b+deg(g))) mod g(X) */
1093 tab = bch->mod8_tab + (b*256+i)*l;
1094 data = i << (8*b);
1095 while (data) {
1096 d = deg(data);
1097 /* subtract X^d.g(X) from p(X).X^(8*b+deg(g)) */
1098 data ^= g[0] >> (31-d);
1099 for (j = 0; j < ecclen; j++) {
1100 hi = (d < 31) ? g[j] << (d+1) : 0;
1101 lo = (j+1 < plen) ?
1102 g[j+1] >> (31-d) : 0;
1103 tab[j] ^= hi|lo;
1104 }
1105 }
1106 }
1107 }
1108}
1109
1110/*
1111 * build a base for factoring degree 2 polynomials
1112 */
1113static int build_deg2_base(struct bch_control *bch)
1114{
1115 const int m = GF_M(bch);
1116 int i, j, r;
1117 unsigned int sum, x, y, remaining, ak = 0, xi[m];
1118
1119 /* find k s.t. Tr(a^k) = 1 and 0 <= k < m */
1120 for (i = 0; i < m; i++) {
1121 for (j = 0, sum = 0; j < m; j++)
1122 sum ^= a_pow(bch, i*(1 << j));
1123
1124 if (sum) {
1125 ak = bch->a_pow_tab[i];
1126 break;
1127 }
1128 }
1129 /* find xi, i=0..m-1 such that xi^2+xi = a^i+Tr(a^i).a^k */
1130 remaining = m;
1131 memset(xi, 0, sizeof(xi));
1132
1133 for (x = 0; (x <= GF_N(bch)) && remaining; x++) {
1134 y = gf_sqr(bch, x)^x;
1135 for (i = 0; i < 2; i++) {
1136 r = a_log(bch, y);
1137 if (y && (r < m) && !xi[r]) {
1138 bch->xi_tab[r] = x;
1139 xi[r] = 1;
1140 remaining--;
1141 dbg("x%d = %x\n", r, x);
1142 break;
1143 }
1144 y ^= ak;
1145 }
1146 }
1147 /* should not happen but check anyway */
1148 return remaining ? -1 : 0;
1149}
1150
1151static void *bch_alloc(size_t size, int *err)
1152{
1153 void *ptr;
1154
1155 ptr = kmalloc(size, GFP_KERNEL);
1156 if (ptr == NULL)
1157 *err = 1;
1158 return ptr;
1159}
1160
1161/*
1162 * compute generator polynomial for given (m,t) parameters.
1163 */
1164static uint32_t *compute_generator_polynomial(struct bch_control *bch)
1165{
1166 const unsigned int m = GF_M(bch);
1167 const unsigned int t = GF_T(bch);
1168 int n, err = 0;
1169 unsigned int i, j, nbits, r, word, *roots;
1170 struct gf_poly *g;
1171 uint32_t *genpoly;
1172
1173 g = bch_alloc(GF_POLY_SZ(m*t), &err);
1174 roots = bch_alloc((bch->n+1)*sizeof(*roots), &err);
1175 genpoly = bch_alloc(DIV_ROUND_UP(m*t+1, 32)*sizeof(*genpoly), &err);
1176
1177 if (err) {
1178 kfree(genpoly);
1179 genpoly = NULL;
1180 goto finish;
1181 }
1182
1183 /* enumerate all roots of g(X) */
1184 memset(roots , 0, (bch->n+1)*sizeof(*roots));
1185 for (i = 0; i < t; i++) {
1186 for (j = 0, r = 2*i+1; j < m; j++) {
1187 roots[r] = 1;
1188 r = mod_s(bch, 2*r);
1189 }
1190 }
1191 /* build generator polynomial g(X) */
1192 g->deg = 0;
1193 g->c[0] = 1;
1194 for (i = 0; i < GF_N(bch); i++) {
1195 if (roots[i]) {
1196 /* multiply g(X) by (X+root) */
1197 r = bch->a_pow_tab[i];
1198 g->c[g->deg+1] = 1;
1199 for (j = g->deg; j > 0; j--)
1200 g->c[j] = gf_mul(bch, g->c[j], r)^g->c[j-1];
1201
1202 g->c[0] = gf_mul(bch, g->c[0], r);
1203 g->deg++;
1204 }
1205 }
1206 /* store left-justified binary representation of g(X) */
1207 n = g->deg+1;
1208 i = 0;
1209
1210 while (n > 0) {
1211 nbits = (n > 32) ? 32 : n;
1212 for (j = 0, word = 0; j < nbits; j++) {
1213 if (g->c[n-1-j])
1214 word |= 1u << (31-j);
1215 }
1216 genpoly[i++] = word;
1217 n -= nbits;
1218 }
1219 bch->ecc_bits = g->deg;
1220
1221finish:
1222 kfree(g);
1223 kfree(roots);
1224
1225 return genpoly;
1226}
1227
1228/**
1229 * init_bch - initialize a BCH encoder/decoder
1230 * @m: Galois field order, should be in the range 5-15
1231 * @t: maximum error correction capability, in bits
1232 * @prim_poly: user-provided primitive polynomial (or 0 to use default)
1233 *
1234 * Returns:
1235 * a newly allocated BCH control structure if successful, NULL otherwise
1236 *
1237 * This initialization can take some time, as lookup tables are built for fast
1238 * encoding/decoding; make sure not to call this function from a time critical
1239 * path. Usually, init_bch() should be called on module/driver init and
1240 * free_bch() should be called to release memory on exit.
1241 *
1242 * You may provide your own primitive polynomial of degree @m in argument
1243 * @prim_poly, or let init_bch() use its default polynomial.
1244 *
1245 * Once init_bch() has successfully returned a pointer to a newly allocated
1246 * BCH control structure, ecc length in bytes is given by member @ecc_bytes of
1247 * the structure.
1248 */
1249struct bch_control *init_bch(int m, int t, unsigned int prim_poly)
1250{
1251 int err = 0;
1252 unsigned int i, words;
1253 uint32_t *genpoly;
1254 struct bch_control *bch = NULL;
1255
1256 const int min_m = 5;
1257 const int max_m = 15;
1258
1259 /* default primitive polynomials */
1260 static const unsigned int prim_poly_tab[] = {
1261 0x25, 0x43, 0x83, 0x11d, 0x211, 0x409, 0x805, 0x1053, 0x201b,
1262 0x402b, 0x8003,
1263 };
1264
1265#if defined(CONFIG_BCH_CONST_PARAMS)
1266 if ((m != (CONFIG_BCH_CONST_M)) || (t != (CONFIG_BCH_CONST_T))) {
1267 printk(KERN_ERR "bch encoder/decoder was configured to support "
1268 "parameters m=%d, t=%d only!\n",
1269 CONFIG_BCH_CONST_M, CONFIG_BCH_CONST_T);
1270 goto fail;
1271 }
1272#endif
1273 if ((m < min_m) || (m > max_m))
1274 /*
1275 * values of m greater than 15 are not currently supported;
1276 * supporting m > 15 would require changing table base type
1277 * (uint16_t) and a small patch in matrix transposition
1278 */
1279 goto fail;
1280
1281 /* sanity checks */
1282 if ((t < 1) || (m*t >= ((1 << m)-1)))
1283 /* invalid t value */
1284 goto fail;
1285
1286 /* select a primitive polynomial for generating GF(2^m) */
1287 if (prim_poly == 0)
1288 prim_poly = prim_poly_tab[m-min_m];
1289
1290 bch = kzalloc(sizeof(*bch), GFP_KERNEL);
1291 if (bch == NULL)
1292 goto fail;
1293
1294 bch->m = m;
1295 bch->t = t;
1296 bch->n = (1 << m)-1;
1297 words = DIV_ROUND_UP(m*t, 32);
1298 bch->ecc_bytes = DIV_ROUND_UP(m*t, 8);
1299 bch->a_pow_tab = bch_alloc((1+bch->n)*sizeof(*bch->a_pow_tab), &err);
1300 bch->a_log_tab = bch_alloc((1+bch->n)*sizeof(*bch->a_log_tab), &err);
1301 bch->mod8_tab = bch_alloc(words*1024*sizeof(*bch->mod8_tab), &err);
1302 bch->ecc_buf = bch_alloc(words*sizeof(*bch->ecc_buf), &err);
1303 bch->ecc_buf2 = bch_alloc(words*sizeof(*bch->ecc_buf2), &err);
1304 bch->xi_tab = bch_alloc(m*sizeof(*bch->xi_tab), &err);
1305 bch->syn = bch_alloc(2*t*sizeof(*bch->syn), &err);
1306 bch->cache = bch_alloc(2*t*sizeof(*bch->cache), &err);
1307 bch->elp = bch_alloc((t+1)*sizeof(struct gf_poly_deg1), &err);
1308
1309 for (i = 0; i < ARRAY_SIZE(bch->poly_2t); i++)
1310 bch->poly_2t[i] = bch_alloc(GF_POLY_SZ(2*t), &err);
1311
1312 if (err)
1313 goto fail;
1314
1315 err = build_gf_tables(bch, prim_poly);
1316 if (err)
1317 goto fail;
1318
1319 /* use generator polynomial for computing encoding tables */
1320 genpoly = compute_generator_polynomial(bch);
1321 if (genpoly == NULL)
1322 goto fail;
1323
1324 build_mod8_tables(bch, genpoly);
1325 kfree(genpoly);
1326
1327 err = build_deg2_base(bch);
1328 if (err)
1329 goto fail;
1330
1331 return bch;
1332
1333fail:
1334 free_bch(bch);
1335 return NULL;
1336}
1337EXPORT_SYMBOL_GPL(init_bch);
1338
1339/**
1340 * free_bch - free the BCH control structure
1341 * @bch: BCH control structure to release
1342 */
1343void free_bch(struct bch_control *bch)
1344{
1345 unsigned int i;
1346
1347 if (bch) {
1348 kfree(bch->a_pow_tab);
1349 kfree(bch->a_log_tab);
1350 kfree(bch->mod8_tab);
1351 kfree(bch->ecc_buf);
1352 kfree(bch->ecc_buf2);
1353 kfree(bch->xi_tab);
1354 kfree(bch->syn);
1355 kfree(bch->cache);
1356 kfree(bch->elp);
1357
1358 for (i = 0; i < ARRAY_SIZE(bch->poly_2t); i++)
1359 kfree(bch->poly_2t[i]);
1360
1361 kfree(bch);
1362 }
1363}
1364EXPORT_SYMBOL_GPL(free_bch);
1365
1366MODULE_LICENSE("GPL");
1367MODULE_AUTHOR("Ivan Djelic <ivan.djelic@parrot.com>");
1368MODULE_DESCRIPTION("Binary BCH encoder/decoder");
diff --git a/lib/bitmap.c b/lib/bitmap.c
index ffb78c916ccd..3f3b68199d74 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -359,7 +359,6 @@ EXPORT_SYMBOL(bitmap_find_next_zero_area);
359 359
360#define CHUNKSZ 32 360#define CHUNKSZ 32
361#define nbits_to_hold_value(val) fls(val) 361#define nbits_to_hold_value(val) fls(val)
362#define unhex(c) (isdigit(c) ? (c - '0') : (toupper(c) - 'A' + 10))
363#define BASEDEC 10 /* fancier cpuset lists input in decimal */ 362#define BASEDEC 10 /* fancier cpuset lists input in decimal */
364 363
365/** 364/**
@@ -466,7 +465,7 @@ int __bitmap_parse(const char *buf, unsigned int buflen,
466 if (chunk & ~((1UL << (CHUNKSZ - 4)) - 1)) 465 if (chunk & ~((1UL << (CHUNKSZ - 4)) - 1))
467 return -EOVERFLOW; 466 return -EOVERFLOW;
468 467
469 chunk = (chunk << 4) | unhex(c); 468 chunk = (chunk << 4) | hex_to_bin(c);
470 ndigits++; totaldigits++; 469 ndigits++; totaldigits++;
471 } 470 }
472 if (ndigits == 0) 471 if (ndigits == 0)
@@ -572,8 +571,11 @@ int bitmap_scnlistprintf(char *buf, unsigned int buflen,
572EXPORT_SYMBOL(bitmap_scnlistprintf); 571EXPORT_SYMBOL(bitmap_scnlistprintf);
573 572
574/** 573/**
575 * bitmap_parselist - convert list format ASCII string to bitmap 574 * __bitmap_parselist - convert list format ASCII string to bitmap
576 * @bp: read nul-terminated user string from this buffer 575 * @buf: read nul-terminated user string from this buffer
576 * @buflen: buffer size in bytes. If string is smaller than this
577 * then it must be terminated with a \0.
578 * @is_user: location of buffer, 0 indicates kernel space
577 * @maskp: write resulting mask here 579 * @maskp: write resulting mask here
578 * @nmaskbits: number of bits in mask to be written 580 * @nmaskbits: number of bits in mask to be written
579 * 581 *
@@ -588,20 +590,63 @@ EXPORT_SYMBOL(bitmap_scnlistprintf);
588 * %-EINVAL: invalid character in string 590 * %-EINVAL: invalid character in string
589 * %-ERANGE: bit number specified too large for mask 591 * %-ERANGE: bit number specified too large for mask
590 */ 592 */
591int bitmap_parselist(const char *bp, unsigned long *maskp, int nmaskbits) 593static int __bitmap_parselist(const char *buf, unsigned int buflen,
594 int is_user, unsigned long *maskp,
595 int nmaskbits)
592{ 596{
593 unsigned a, b; 597 unsigned a, b;
598 int c, old_c, totaldigits;
599 const char __user *ubuf = buf;
600 int exp_digit, in_range;
594 601
602 totaldigits = c = 0;
595 bitmap_zero(maskp, nmaskbits); 603 bitmap_zero(maskp, nmaskbits);
596 do { 604 do {
597 if (!isdigit(*bp)) 605 exp_digit = 1;
598 return -EINVAL; 606 in_range = 0;
599 b = a = simple_strtoul(bp, (char **)&bp, BASEDEC); 607 a = b = 0;
600 if (*bp == '-') { 608
601 bp++; 609 /* Get the next cpu# or a range of cpu#'s */
602 if (!isdigit(*bp)) 610 while (buflen) {
611 old_c = c;
612 if (is_user) {
613 if (__get_user(c, ubuf++))
614 return -EFAULT;
615 } else
616 c = *buf++;
617 buflen--;
618 if (isspace(c))
619 continue;
620
621 /*
622 * If the last character was a space and the current
623 * character isn't '\0', we've got embedded whitespace.
624 * This is a no-no, so throw an error.
625 */
626 if (totaldigits && c && isspace(old_c))
627 return -EINVAL;
628
629 /* A '\0' or a ',' signal the end of a cpu# or range */
630 if (c == '\0' || c == ',')
631 break;
632
633 if (c == '-') {
634 if (exp_digit || in_range)
635 return -EINVAL;
636 b = 0;
637 in_range = 1;
638 exp_digit = 1;
639 continue;
640 }
641
642 if (!isdigit(c))
603 return -EINVAL; 643 return -EINVAL;
604 b = simple_strtoul(bp, (char **)&bp, BASEDEC); 644
645 b = b * 10 + (c - '0');
646 if (!in_range)
647 a = b;
648 exp_digit = 0;
649 totaldigits++;
605 } 650 }
606 if (!(a <= b)) 651 if (!(a <= b))
607 return -EINVAL; 652 return -EINVAL;
@@ -611,13 +656,52 @@ int bitmap_parselist(const char *bp, unsigned long *maskp, int nmaskbits)
611 set_bit(a, maskp); 656 set_bit(a, maskp);
612 a++; 657 a++;
613 } 658 }
614 if (*bp == ',') 659 } while (buflen && c == ',');
615 bp++;
616 } while (*bp != '\0' && *bp != '\n');
617 return 0; 660 return 0;
618} 661}
662
663int bitmap_parselist(const char *bp, unsigned long *maskp, int nmaskbits)
664{
665 char *nl = strchr(bp, '\n');
666 int len;
667
668 if (nl)
669 len = nl - bp;
670 else
671 len = strlen(bp);
672
673 return __bitmap_parselist(bp, len, 0, maskp, nmaskbits);
674}
619EXPORT_SYMBOL(bitmap_parselist); 675EXPORT_SYMBOL(bitmap_parselist);
620 676
677
678/**
679 * bitmap_parselist_user()
680 *
681 * @ubuf: pointer to user buffer containing string.
682 * @ulen: buffer size in bytes. If string is smaller than this
683 * then it must be terminated with a \0.
684 * @maskp: pointer to bitmap array that will contain result.
685 * @nmaskbits: size of bitmap, in bits.
686 *
687 * Wrapper for bitmap_parselist(), providing it with user buffer.
688 *
689 * We cannot have this as an inline function in bitmap.h because it needs
690 * linux/uaccess.h to get the access_ok() declaration and this causes
691 * cyclic dependencies.
692 */
693int bitmap_parselist_user(const char __user *ubuf,
694 unsigned int ulen, unsigned long *maskp,
695 int nmaskbits)
696{
697 if (!access_ok(VERIFY_READ, ubuf, ulen))
698 return -EFAULT;
699 return __bitmap_parselist((const char *)ubuf,
700 ulen, 1, maskp, nmaskbits);
701}
702EXPORT_SYMBOL(bitmap_parselist_user);
703
704
621/** 705/**
622 * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap 706 * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap
623 * @buf: pointer to a bitmap 707 * @buf: pointer to a bitmap
@@ -831,7 +915,7 @@ EXPORT_SYMBOL(bitmap_bitremap);
831 * @orig (i.e. bits 3, 5, 7 and 9) were also set. 915 * @orig (i.e. bits 3, 5, 7 and 9) were also set.
832 * 916 *
833 * When bit 11 is set in @orig, it means turn on the bit in 917 * When bit 11 is set in @orig, it means turn on the bit in
834 * @dst corresponding to whatever is the twelth bit that is 918 * @dst corresponding to whatever is the twelfth bit that is
835 * turned on in @relmap. In the above example, there were 919 * turned on in @relmap. In the above example, there were
836 * only ten bits turned on in @relmap (30..39), so that bit 920 * only ten bits turned on in @relmap (30..39), so that bit
837 * 11 was set in @orig had no affect on @dst. 921 * 11 was set in @orig had no affect on @dst.
diff --git a/lib/bsearch.c b/lib/bsearch.c
new file mode 100644
index 000000000000..5b54758e2afb
--- /dev/null
+++ b/lib/bsearch.c
@@ -0,0 +1,53 @@
1/*
2 * A generic implementation of binary search for the Linux kernel
3 *
4 * Copyright (C) 2008-2009 Ksplice, Inc.
5 * Author: Tim Abbott <tabbott@ksplice.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; version 2.
10 */
11
12#include <linux/module.h>
13#include <linux/bsearch.h>
14
15/*
16 * bsearch - binary search an array of elements
17 * @key: pointer to item being searched for
18 * @base: pointer to first element to search
19 * @num: number of elements
20 * @size: size of each element
21 * @cmp: pointer to comparison function
22 *
23 * This function does a binary search on the given array. The
24 * contents of the array should already be in ascending sorted order
25 * under the provided comparison function.
26 *
27 * Note that the key need not have the same type as the elements in
28 * the array, e.g. key could be a string and the comparison function
29 * could compare the string with the struct's name field. However, if
30 * the key and elements in the array are of the same type, you can use
31 * the same comparison function for both sort() and bsearch().
32 */
33void *bsearch(const void *key, const void *base, size_t num, size_t size,
34 int (*cmp)(const void *key, const void *elt))
35{
36 size_t start = 0, end = num;
37 int result;
38
39 while (start < end) {
40 size_t mid = start + (end - start) / 2;
41
42 result = cmp(key, base + mid * size);
43 if (result < 0)
44 end = mid;
45 else if (result > 0)
46 start = mid + 1;
47 else
48 return (void *)base + mid * size;
49 }
50
51 return NULL;
52}
53EXPORT_SYMBOL(bsearch);
diff --git a/lib/btree.c b/lib/btree.c
index c9c6f0351526..2a34392bcecc 100644
--- a/lib/btree.c
+++ b/lib/btree.c
@@ -11,7 +11,7 @@
11 * see http://programming.kicks-ass.net/kernel-patches/vma_lookup/btree.patch 11 * see http://programming.kicks-ass.net/kernel-patches/vma_lookup/btree.patch
12 * 12 *
13 * A relatively simple B+Tree implementation. I have written it as a learning 13 * A relatively simple B+Tree implementation. I have written it as a learning
14 * excercise to understand how B+Trees work. Turned out to be useful as well. 14 * exercise to understand how B+Trees work. Turned out to be useful as well.
15 * 15 *
16 * B+Trees can be used similar to Linux radix trees (which don't have anything 16 * B+Trees can be used similar to Linux radix trees (which don't have anything
17 * in common with textbook radix trees, beware). Prerequisite for them working 17 * in common with textbook radix trees, beware). Prerequisite for them working
@@ -541,7 +541,7 @@ static void rebalance(struct btree_head *head, struct btree_geo *geo,
541 int i, no_left, no_right; 541 int i, no_left, no_right;
542 542
543 if (fill == 0) { 543 if (fill == 0) {
544 /* Because we don't steal entries from a neigbour, this case 544 /* Because we don't steal entries from a neighbour, this case
545 * can happen. Parent node contains a single child, this 545 * can happen. Parent node contains a single child, this
546 * node, so merging with a sibling never happens. 546 * node, so merging with a sibling never happens.
547 */ 547 */
diff --git a/lib/cpu_rmap.c b/lib/cpu_rmap.c
new file mode 100644
index 000000000000..987acfafeb83
--- /dev/null
+++ b/lib/cpu_rmap.c
@@ -0,0 +1,269 @@
1/*
2 * cpu_rmap.c: CPU affinity reverse-map support
3 * Copyright 2011 Solarflare Communications Inc.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation, incorporated herein by reference.
8 */
9
10#include <linux/cpu_rmap.h>
11#ifdef CONFIG_GENERIC_HARDIRQS
12#include <linux/interrupt.h>
13#endif
14#include <linux/module.h>
15
16/*
17 * These functions maintain a mapping from CPUs to some ordered set of
18 * objects with CPU affinities. This can be seen as a reverse-map of
19 * CPU affinity. However, we do not assume that the object affinities
20 * cover all CPUs in the system. For those CPUs not directly covered
21 * by object affinities, we attempt to find a nearest object based on
22 * CPU topology.
23 */
24
25/**
26 * alloc_cpu_rmap - allocate CPU affinity reverse-map
27 * @size: Number of objects to be mapped
28 * @flags: Allocation flags e.g. %GFP_KERNEL
29 */
30struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags)
31{
32 struct cpu_rmap *rmap;
33 unsigned int cpu;
34 size_t obj_offset;
35
36 /* This is a silly number of objects, and we use u16 indices. */
37 if (size > 0xffff)
38 return NULL;
39
40 /* Offset of object pointer array from base structure */
41 obj_offset = ALIGN(offsetof(struct cpu_rmap, near[nr_cpu_ids]),
42 sizeof(void *));
43
44 rmap = kzalloc(obj_offset + size * sizeof(rmap->obj[0]), flags);
45 if (!rmap)
46 return NULL;
47
48 rmap->obj = (void **)((char *)rmap + obj_offset);
49
50 /* Initially assign CPUs to objects on a rota, since we have
51 * no idea where the objects are. Use infinite distance, so
52 * any object with known distance is preferable. Include the
53 * CPUs that are not present/online, since we definitely want
54 * any newly-hotplugged CPUs to have some object assigned.
55 */
56 for_each_possible_cpu(cpu) {
57 rmap->near[cpu].index = cpu % size;
58 rmap->near[cpu].dist = CPU_RMAP_DIST_INF;
59 }
60
61 rmap->size = size;
62 return rmap;
63}
64EXPORT_SYMBOL(alloc_cpu_rmap);
65
66/* Reevaluate nearest object for given CPU, comparing with the given
67 * neighbours at the given distance.
68 */
69static bool cpu_rmap_copy_neigh(struct cpu_rmap *rmap, unsigned int cpu,
70 const struct cpumask *mask, u16 dist)
71{
72 int neigh;
73
74 for_each_cpu(neigh, mask) {
75 if (rmap->near[cpu].dist > dist &&
76 rmap->near[neigh].dist <= dist) {
77 rmap->near[cpu].index = rmap->near[neigh].index;
78 rmap->near[cpu].dist = dist;
79 return true;
80 }
81 }
82 return false;
83}
84
85#ifdef DEBUG
86static void debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
87{
88 unsigned index;
89 unsigned int cpu;
90
91 pr_info("cpu_rmap %p, %s:\n", rmap, prefix);
92
93 for_each_possible_cpu(cpu) {
94 index = rmap->near[cpu].index;
95 pr_info("cpu %d -> obj %u (distance %u)\n",
96 cpu, index, rmap->near[cpu].dist);
97 }
98}
99#else
100static inline void
101debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
102{
103}
104#endif
105
106/**
107 * cpu_rmap_add - add object to a rmap
108 * @rmap: CPU rmap allocated with alloc_cpu_rmap()
109 * @obj: Object to add to rmap
110 *
111 * Return index of object.
112 */
113int cpu_rmap_add(struct cpu_rmap *rmap, void *obj)
114{
115 u16 index;
116
117 BUG_ON(rmap->used >= rmap->size);
118 index = rmap->used++;
119 rmap->obj[index] = obj;
120 return index;
121}
122EXPORT_SYMBOL(cpu_rmap_add);
123
124/**
125 * cpu_rmap_update - update CPU rmap following a change of object affinity
126 * @rmap: CPU rmap to update
127 * @index: Index of object whose affinity changed
128 * @affinity: New CPU affinity of object
129 */
130int cpu_rmap_update(struct cpu_rmap *rmap, u16 index,
131 const struct cpumask *affinity)
132{
133 cpumask_var_t update_mask;
134 unsigned int cpu;
135
136 if (unlikely(!zalloc_cpumask_var(&update_mask, GFP_KERNEL)))
137 return -ENOMEM;
138
139 /* Invalidate distance for all CPUs for which this used to be
140 * the nearest object. Mark those CPUs for update.
141 */
142 for_each_online_cpu(cpu) {
143 if (rmap->near[cpu].index == index) {
144 rmap->near[cpu].dist = CPU_RMAP_DIST_INF;
145 cpumask_set_cpu(cpu, update_mask);
146 }
147 }
148
149 debug_print_rmap(rmap, "after invalidating old distances");
150
151 /* Set distance to 0 for all CPUs in the new affinity mask.
152 * Mark all CPUs within their NUMA nodes for update.
153 */
154 for_each_cpu(cpu, affinity) {
155 rmap->near[cpu].index = index;
156 rmap->near[cpu].dist = 0;
157 cpumask_or(update_mask, update_mask,
158 cpumask_of_node(cpu_to_node(cpu)));
159 }
160
161 debug_print_rmap(rmap, "after updating neighbours");
162
163 /* Update distances based on topology */
164 for_each_cpu(cpu, update_mask) {
165 if (cpu_rmap_copy_neigh(rmap, cpu,
166 topology_thread_cpumask(cpu), 1))
167 continue;
168 if (cpu_rmap_copy_neigh(rmap, cpu,
169 topology_core_cpumask(cpu), 2))
170 continue;
171 if (cpu_rmap_copy_neigh(rmap, cpu,
172 cpumask_of_node(cpu_to_node(cpu)), 3))
173 continue;
174 /* We could continue into NUMA node distances, but for now
175 * we give up.
176 */
177 }
178
179 debug_print_rmap(rmap, "after copying neighbours");
180
181 free_cpumask_var(update_mask);
182 return 0;
183}
184EXPORT_SYMBOL(cpu_rmap_update);
185
186#ifdef CONFIG_GENERIC_HARDIRQS
187
188/* Glue between IRQ affinity notifiers and CPU rmaps */
189
190struct irq_glue {
191 struct irq_affinity_notify notify;
192 struct cpu_rmap *rmap;
193 u16 index;
194};
195
196/**
197 * free_irq_cpu_rmap - free a CPU affinity reverse-map used for IRQs
198 * @rmap: Reverse-map allocated with alloc_irq_cpu_map(), or %NULL
199 *
200 * Must be called in process context, before freeing the IRQs, and
201 * without holding any locks required by global workqueue items.
202 */
203void free_irq_cpu_rmap(struct cpu_rmap *rmap)
204{
205 struct irq_glue *glue;
206 u16 index;
207
208 if (!rmap)
209 return;
210
211 for (index = 0; index < rmap->used; index++) {
212 glue = rmap->obj[index];
213 irq_set_affinity_notifier(glue->notify.irq, NULL);
214 }
215 irq_run_affinity_notifiers();
216
217 kfree(rmap);
218}
219EXPORT_SYMBOL(free_irq_cpu_rmap);
220
221static void
222irq_cpu_rmap_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
223{
224 struct irq_glue *glue =
225 container_of(notify, struct irq_glue, notify);
226 int rc;
227
228 rc = cpu_rmap_update(glue->rmap, glue->index, mask);
229 if (rc)
230 pr_warning("irq_cpu_rmap_notify: update failed: %d\n", rc);
231}
232
233static void irq_cpu_rmap_release(struct kref *ref)
234{
235 struct irq_glue *glue =
236 container_of(ref, struct irq_glue, notify.kref);
237 kfree(glue);
238}
239
240/**
241 * irq_cpu_rmap_add - add an IRQ to a CPU affinity reverse-map
242 * @rmap: The reverse-map
243 * @irq: The IRQ number
244 *
245 * This adds an IRQ affinity notifier that will update the reverse-map
246 * automatically.
247 *
248 * Must be called in process context, after the IRQ is allocated but
249 * before it is bound with request_irq().
250 */
251int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq)
252{
253 struct irq_glue *glue = kzalloc(sizeof(*glue), GFP_KERNEL);
254 int rc;
255
256 if (!glue)
257 return -ENOMEM;
258 glue->notify.notify = irq_cpu_rmap_notify;
259 glue->notify.release = irq_cpu_rmap_release;
260 glue->rmap = rmap;
261 glue->index = cpu_rmap_add(rmap, glue);
262 rc = irq_set_affinity_notifier(irq, &glue->notify);
263 if (rc)
264 kfree(glue);
265 return rc;
266}
267EXPORT_SYMBOL(irq_cpu_rmap_add);
268
269#endif /* CONFIG_GENERIC_HARDIRQS */
diff --git a/lib/debug_locks.c b/lib/debug_locks.c
index 5bf0020b9248..b1c177307677 100644
--- a/lib/debug_locks.c
+++ b/lib/debug_locks.c
@@ -8,7 +8,6 @@
8 * 8 *
9 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 9 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
10 */ 10 */
11#include <linux/kernel.h>
12#include <linux/rwsem.h> 11#include <linux/rwsem.h>
13#include <linux/mutex.h> 12#include <linux/mutex.h>
14#include <linux/module.h> 13#include <linux/module.h>
@@ -39,7 +38,6 @@ int debug_locks_off(void)
39{ 38{
40 if (__debug_locks_off()) { 39 if (__debug_locks_off()) {
41 if (!debug_locks_silent) { 40 if (!debug_locks_silent) {
42 oops_in_progress = 1;
43 console_verbose(); 41 console_verbose();
44 return 1; 42 return 1;
45 } 43 }
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index deebcc57d4e6..a78b7c6e042c 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -198,7 +198,7 @@ static void free_object(struct debug_obj *obj)
198 * initialized: 198 * initialized:
199 */ 199 */
200 if (obj_pool_free > ODEBUG_POOL_SIZE && obj_cache) 200 if (obj_pool_free > ODEBUG_POOL_SIZE && obj_cache)
201 sched = !work_pending(&debug_obj_work); 201 sched = keventd_up() && !work_pending(&debug_obj_work);
202 hlist_add_head(&obj->node, &obj_pool); 202 hlist_add_head(&obj->node, &obj_pool);
203 obj_pool_free++; 203 obj_pool_free++;
204 obj_pool_used--; 204 obj_pool_used--;
@@ -249,14 +249,17 @@ static struct debug_bucket *get_bucket(unsigned long addr)
249 249
250static void debug_print_object(struct debug_obj *obj, char *msg) 250static void debug_print_object(struct debug_obj *obj, char *msg)
251{ 251{
252 struct debug_obj_descr *descr = obj->descr;
252 static int limit; 253 static int limit;
253 254
254 if (limit < 5 && obj->descr != descr_test) { 255 if (limit < 5 && descr != descr_test) {
256 void *hint = descr->debug_hint ?
257 descr->debug_hint(obj->object) : NULL;
255 limit++; 258 limit++;
256 WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) " 259 WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
257 "object type: %s\n", 260 "object type: %s hint: %pS\n",
258 msg, obj_states[obj->state], obj->astate, 261 msg, obj_states[obj->state], obj->astate,
259 obj->descr->name); 262 descr->name, hint);
260 } 263 }
261 debug_objects_warnings++; 264 debug_objects_warnings++;
262} 265}
diff --git a/lib/decompress.c b/lib/decompress.c
index a7606815541f..3d766b7f60ab 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -8,6 +8,7 @@
8 8
9#include <linux/decompress/bunzip2.h> 9#include <linux/decompress/bunzip2.h>
10#include <linux/decompress/unlzma.h> 10#include <linux/decompress/unlzma.h>
11#include <linux/decompress/unxz.h>
11#include <linux/decompress/inflate.h> 12#include <linux/decompress/inflate.h>
12#include <linux/decompress/unlzo.h> 13#include <linux/decompress/unlzo.h>
13 14
@@ -23,6 +24,9 @@
23#ifndef CONFIG_DECOMPRESS_LZMA 24#ifndef CONFIG_DECOMPRESS_LZMA
24# define unlzma NULL 25# define unlzma NULL
25#endif 26#endif
27#ifndef CONFIG_DECOMPRESS_XZ
28# define unxz NULL
29#endif
26#ifndef CONFIG_DECOMPRESS_LZO 30#ifndef CONFIG_DECOMPRESS_LZO
27# define unlzo NULL 31# define unlzo NULL
28#endif 32#endif
@@ -36,6 +40,7 @@ static const struct compress_format {
36 { {037, 0236}, "gzip", gunzip }, 40 { {037, 0236}, "gzip", gunzip },
37 { {0x42, 0x5a}, "bzip2", bunzip2 }, 41 { {0x42, 0x5a}, "bzip2", bunzip2 },
38 { {0x5d, 0x00}, "lzma", unlzma }, 42 { {0x5d, 0x00}, "lzma", unlzma },
43 { {0xfd, 0x37}, "xz", unxz },
39 { {0x89, 0x4c}, "lzo", unlzo }, 44 { {0x89, 0x4c}, "lzo", unlzo },
40 { {0, 0}, NULL, NULL } 45 { {0, 0}, NULL, NULL }
41}; 46};
diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c
index 81c8bb1cc6aa..a7b80c1d6a0d 100644
--- a/lib/decompress_bunzip2.c
+++ b/lib/decompress_bunzip2.c
@@ -49,7 +49,6 @@
49#define PREBOOT 49#define PREBOOT
50#else 50#else
51#include <linux/decompress/bunzip2.h> 51#include <linux/decompress/bunzip2.h>
52#include <linux/slab.h>
53#endif /* STATIC */ 52#endif /* STATIC */
54 53
55#include <linux/decompress/mm.h> 54#include <linux/decompress/mm.h>
@@ -682,13 +681,12 @@ STATIC int INIT bunzip2(unsigned char *buf, int len,
682 int(*flush)(void*, unsigned int), 681 int(*flush)(void*, unsigned int),
683 unsigned char *outbuf, 682 unsigned char *outbuf,
684 int *pos, 683 int *pos,
685 void(*error_fn)(char *x)) 684 void(*error)(char *x))
686{ 685{
687 struct bunzip_data *bd; 686 struct bunzip_data *bd;
688 int i = -1; 687 int i = -1;
689 unsigned char *inbuf; 688 unsigned char *inbuf;
690 689
691 set_error_fn(error_fn);
692 if (flush) 690 if (flush)
693 outbuf = malloc(BZIP2_IOBUF_SIZE); 691 outbuf = malloc(BZIP2_IOBUF_SIZE);
694 692
@@ -751,8 +749,8 @@ STATIC int INIT decompress(unsigned char *buf, int len,
751 int(*flush)(void*, unsigned int), 749 int(*flush)(void*, unsigned int),
752 unsigned char *outbuf, 750 unsigned char *outbuf,
753 int *pos, 751 int *pos,
754 void(*error_fn)(char *x)) 752 void(*error)(char *x))
755{ 753{
756 return bunzip2(buf, len - 4, fill, flush, outbuf, pos, error_fn); 754 return bunzip2(buf, len - 4, fill, flush, outbuf, pos, error);
757} 755}
758#endif 756#endif
diff --git a/lib/decompress_inflate.c b/lib/decompress_inflate.c
index fc686c7a0a0d..19ff89e34eec 100644
--- a/lib/decompress_inflate.c
+++ b/lib/decompress_inflate.c
@@ -19,7 +19,6 @@
19#include "zlib_inflate/inflate.h" 19#include "zlib_inflate/inflate.h"
20 20
21#include "zlib_inflate/infutil.h" 21#include "zlib_inflate/infutil.h"
22#include <linux/slab.h>
23 22
24#endif /* STATIC */ 23#endif /* STATIC */
25 24
@@ -27,7 +26,7 @@
27 26
28#define GZIP_IOBUF_SIZE (16*1024) 27#define GZIP_IOBUF_SIZE (16*1024)
29 28
30static int nofill(void *buffer, unsigned int len) 29static int INIT nofill(void *buffer, unsigned int len)
31{ 30{
32 return -1; 31 return -1;
33} 32}
@@ -38,13 +37,12 @@ STATIC int INIT gunzip(unsigned char *buf, int len,
38 int(*flush)(void*, unsigned int), 37 int(*flush)(void*, unsigned int),
39 unsigned char *out_buf, 38 unsigned char *out_buf,
40 int *pos, 39 int *pos,
41 void(*error_fn)(char *x)) { 40 void(*error)(char *x)) {
42 u8 *zbuf; 41 u8 *zbuf;
43 struct z_stream_s *strm; 42 struct z_stream_s *strm;
44 int rc; 43 int rc;
45 size_t out_len; 44 size_t out_len;
46 45
47 set_error_fn(error_fn);
48 rc = -1; 46 rc = -1;
49 if (flush) { 47 if (flush) {
50 out_len = 0x8000; /* 32 K */ 48 out_len = 0x8000; /* 32 K */
@@ -100,13 +98,22 @@ STATIC int INIT gunzip(unsigned char *buf, int len,
100 * possible asciz filename) 98 * possible asciz filename)
101 */ 99 */
102 strm->next_in = zbuf + 10; 100 strm->next_in = zbuf + 10;
101 strm->avail_in = len - 10;
103 /* skip over asciz filename */ 102 /* skip over asciz filename */
104 if (zbuf[3] & 0x8) { 103 if (zbuf[3] & 0x8) {
105 while (strm->next_in[0]) 104 do {
106 strm->next_in++; 105 /*
107 strm->next_in++; 106 * If the filename doesn't fit into the buffer,
107 * the file is very probably corrupt. Don't try
108 * to read more data.
109 */
110 if (strm->avail_in == 0) {
111 error("header error");
112 goto gunzip_5;
113 }
114 --strm->avail_in;
115 } while (*strm->next_in++);
108 } 116 }
109 strm->avail_in = len - (strm->next_in - zbuf);
110 117
111 strm->next_out = out_buf; 118 strm->next_out = out_buf;
112 strm->avail_out = out_len; 119 strm->avail_out = out_len;
diff --git a/lib/decompress_unlzma.c b/lib/decompress_unlzma.c
index ca82fde81c8f..476c65af9709 100644
--- a/lib/decompress_unlzma.c
+++ b/lib/decompress_unlzma.c
@@ -33,7 +33,6 @@
33#define PREBOOT 33#define PREBOOT
34#else 34#else
35#include <linux/decompress/unlzma.h> 35#include <linux/decompress/unlzma.h>
36#include <linux/slab.h>
37#endif /* STATIC */ 36#endif /* STATIC */
38 37
39#include <linux/decompress/mm.h> 38#include <linux/decompress/mm.h>
@@ -74,6 +73,7 @@ struct rc {
74 uint32_t code; 73 uint32_t code;
75 uint32_t range; 74 uint32_t range;
76 uint32_t bound; 75 uint32_t bound;
76 void (*error)(char *);
77}; 77};
78 78
79 79
@@ -82,7 +82,7 @@ struct rc {
82#define RC_MODEL_TOTAL_BITS 11 82#define RC_MODEL_TOTAL_BITS 11
83 83
84 84
85static int nofill(void *buffer, unsigned int len) 85static int INIT nofill(void *buffer, unsigned int len)
86{ 86{
87 return -1; 87 return -1;
88} 88}
@@ -92,7 +92,7 @@ static void INIT rc_read(struct rc *rc)
92{ 92{
93 rc->buffer_size = rc->fill((char *)rc->buffer, LZMA_IOBUF_SIZE); 93 rc->buffer_size = rc->fill((char *)rc->buffer, LZMA_IOBUF_SIZE);
94 if (rc->buffer_size <= 0) 94 if (rc->buffer_size <= 0)
95 error("unexpected EOF"); 95 rc->error("unexpected EOF");
96 rc->ptr = rc->buffer; 96 rc->ptr = rc->buffer;
97 rc->buffer_end = rc->buffer + rc->buffer_size; 97 rc->buffer_end = rc->buffer + rc->buffer_size;
98} 98}
@@ -127,12 +127,6 @@ static inline void INIT rc_init_code(struct rc *rc)
127} 127}
128 128
129 129
130/* Called once. TODO: bb_maybe_free() */
131static inline void INIT rc_free(struct rc *rc)
132{
133 free(rc->buffer);
134}
135
136/* Called twice, but one callsite is in inline'd rc_is_bit_0_helper() */ 130/* Called twice, but one callsite is in inline'd rc_is_bit_0_helper() */
137static void INIT rc_do_normalize(struct rc *rc) 131static void INIT rc_do_normalize(struct rc *rc)
138{ 132{
@@ -169,7 +163,7 @@ static inline void INIT rc_update_bit_0(struct rc *rc, uint16_t *p)
169 rc->range = rc->bound; 163 rc->range = rc->bound;
170 *p += ((1 << RC_MODEL_TOTAL_BITS) - *p) >> RC_MOVE_BITS; 164 *p += ((1 << RC_MODEL_TOTAL_BITS) - *p) >> RC_MOVE_BITS;
171} 165}
172static inline void rc_update_bit_1(struct rc *rc, uint16_t *p) 166static inline void INIT rc_update_bit_1(struct rc *rc, uint16_t *p)
173{ 167{
174 rc->range -= rc->bound; 168 rc->range -= rc->bound;
175 rc->code -= rc->bound; 169 rc->code -= rc->bound;
@@ -319,32 +313,38 @@ static inline uint8_t INIT peek_old_byte(struct writer *wr,
319 313
320} 314}
321 315
322static inline void INIT write_byte(struct writer *wr, uint8_t byte) 316static inline int INIT write_byte(struct writer *wr, uint8_t byte)
323{ 317{
324 wr->buffer[wr->buffer_pos++] = wr->previous_byte = byte; 318 wr->buffer[wr->buffer_pos++] = wr->previous_byte = byte;
325 if (wr->flush && wr->buffer_pos == wr->header->dict_size) { 319 if (wr->flush && wr->buffer_pos == wr->header->dict_size) {
326 wr->buffer_pos = 0; 320 wr->buffer_pos = 0;
327 wr->global_pos += wr->header->dict_size; 321 wr->global_pos += wr->header->dict_size;
328 wr->flush((char *)wr->buffer, wr->header->dict_size); 322 if (wr->flush((char *)wr->buffer, wr->header->dict_size)
323 != wr->header->dict_size)
324 return -1;
329 } 325 }
326 return 0;
330} 327}
331 328
332 329
333static inline void INIT copy_byte(struct writer *wr, uint32_t offs) 330static inline int INIT copy_byte(struct writer *wr, uint32_t offs)
334{ 331{
335 write_byte(wr, peek_old_byte(wr, offs)); 332 return write_byte(wr, peek_old_byte(wr, offs));
336} 333}
337 334
338static inline void INIT copy_bytes(struct writer *wr, 335static inline int INIT copy_bytes(struct writer *wr,
339 uint32_t rep0, int len) 336 uint32_t rep0, int len)
340{ 337{
341 do { 338 do {
342 copy_byte(wr, rep0); 339 if (copy_byte(wr, rep0))
340 return -1;
343 len--; 341 len--;
344 } while (len != 0 && wr->buffer_pos < wr->header->dst_size); 342 } while (len != 0 && wr->buffer_pos < wr->header->dst_size);
343
344 return len;
345} 345}
346 346
347static inline void INIT process_bit0(struct writer *wr, struct rc *rc, 347static inline int INIT process_bit0(struct writer *wr, struct rc *rc,
348 struct cstate *cst, uint16_t *p, 348 struct cstate *cst, uint16_t *p,
349 int pos_state, uint16_t *prob, 349 int pos_state, uint16_t *prob,
350 int lc, uint32_t literal_pos_mask) { 350 int lc, uint32_t literal_pos_mask) {
@@ -378,16 +378,17 @@ static inline void INIT process_bit0(struct writer *wr, struct rc *rc,
378 uint16_t *prob_lit = prob + mi; 378 uint16_t *prob_lit = prob + mi;
379 rc_get_bit(rc, prob_lit, &mi); 379 rc_get_bit(rc, prob_lit, &mi);
380 } 380 }
381 write_byte(wr, mi);
382 if (cst->state < 4) 381 if (cst->state < 4)
383 cst->state = 0; 382 cst->state = 0;
384 else if (cst->state < 10) 383 else if (cst->state < 10)
385 cst->state -= 3; 384 cst->state -= 3;
386 else 385 else
387 cst->state -= 6; 386 cst->state -= 6;
387
388 return write_byte(wr, mi);
388} 389}
389 390
390static inline void INIT process_bit1(struct writer *wr, struct rc *rc, 391static inline int INIT process_bit1(struct writer *wr, struct rc *rc,
391 struct cstate *cst, uint16_t *p, 392 struct cstate *cst, uint16_t *p,
392 int pos_state, uint16_t *prob) { 393 int pos_state, uint16_t *prob) {
393 int offset; 394 int offset;
@@ -418,8 +419,7 @@ static inline void INIT process_bit1(struct writer *wr, struct rc *rc,
418 419
419 cst->state = cst->state < LZMA_NUM_LIT_STATES ? 420 cst->state = cst->state < LZMA_NUM_LIT_STATES ?
420 9 : 11; 421 9 : 11;
421 copy_byte(wr, cst->rep0); 422 return copy_byte(wr, cst->rep0);
422 return;
423 } else { 423 } else {
424 rc_update_bit_1(rc, prob); 424 rc_update_bit_1(rc, prob);
425 } 425 }
@@ -521,12 +521,15 @@ static inline void INIT process_bit1(struct writer *wr, struct rc *rc,
521 } else 521 } else
522 cst->rep0 = pos_slot; 522 cst->rep0 = pos_slot;
523 if (++(cst->rep0) == 0) 523 if (++(cst->rep0) == 0)
524 return; 524 return 0;
525 if (cst->rep0 > wr->header->dict_size
526 || cst->rep0 > get_pos(wr))
527 return -1;
525 } 528 }
526 529
527 len += LZMA_MATCH_MIN_LEN; 530 len += LZMA_MATCH_MIN_LEN;
528 531
529 copy_bytes(wr, cst->rep0, len); 532 return copy_bytes(wr, cst->rep0, len);
530} 533}
531 534
532 535
@@ -536,7 +539,7 @@ STATIC inline int INIT unlzma(unsigned char *buf, int in_len,
536 int(*flush)(void*, unsigned int), 539 int(*flush)(void*, unsigned int),
537 unsigned char *output, 540 unsigned char *output,
538 int *posp, 541 int *posp,
539 void(*error_fn)(char *x) 542 void(*error)(char *x)
540 ) 543 )
541{ 544{
542 struct lzma_header header; 545 struct lzma_header header;
@@ -552,7 +555,7 @@ STATIC inline int INIT unlzma(unsigned char *buf, int in_len,
552 unsigned char *inbuf; 555 unsigned char *inbuf;
553 int ret = -1; 556 int ret = -1;
554 557
555 set_error_fn(error_fn); 558 rc.error = error;
556 559
557 if (buf) 560 if (buf)
558 inbuf = buf; 561 inbuf = buf;
@@ -580,8 +583,10 @@ STATIC inline int INIT unlzma(unsigned char *buf, int in_len,
580 ((unsigned char *)&header)[i] = *rc.ptr++; 583 ((unsigned char *)&header)[i] = *rc.ptr++;
581 } 584 }
582 585
583 if (header.pos >= (9 * 5 * 5)) 586 if (header.pos >= (9 * 5 * 5)) {
584 error("bad header"); 587 error("bad header");
588 goto exit_1;
589 }
585 590
586 mi = 0; 591 mi = 0;
587 lc = header.pos; 592 lc = header.pos;
@@ -627,21 +632,29 @@ STATIC inline int INIT unlzma(unsigned char *buf, int in_len,
627 int pos_state = get_pos(&wr) & pos_state_mask; 632 int pos_state = get_pos(&wr) & pos_state_mask;
628 uint16_t *prob = p + LZMA_IS_MATCH + 633 uint16_t *prob = p + LZMA_IS_MATCH +
629 (cst.state << LZMA_NUM_POS_BITS_MAX) + pos_state; 634 (cst.state << LZMA_NUM_POS_BITS_MAX) + pos_state;
630 if (rc_is_bit_0(&rc, prob)) 635 if (rc_is_bit_0(&rc, prob)) {
631 process_bit0(&wr, &rc, &cst, p, pos_state, prob, 636 if (process_bit0(&wr, &rc, &cst, p, pos_state, prob,
632 lc, literal_pos_mask); 637 lc, literal_pos_mask)) {
633 else { 638 error("LZMA data is corrupt");
634 process_bit1(&wr, &rc, &cst, p, pos_state, prob); 639 goto exit_3;
640 }
641 } else {
642 if (process_bit1(&wr, &rc, &cst, p, pos_state, prob)) {
643 error("LZMA data is corrupt");
644 goto exit_3;
645 }
635 if (cst.rep0 == 0) 646 if (cst.rep0 == 0)
636 break; 647 break;
637 } 648 }
649 if (rc.buffer_size <= 0)
650 goto exit_3;
638 } 651 }
639 652
640 if (posp) 653 if (posp)
641 *posp = rc.ptr-rc.buffer; 654 *posp = rc.ptr-rc.buffer;
642 if (wr.flush) 655 if (!wr.flush || wr.flush(wr.buffer, wr.buffer_pos) == wr.buffer_pos)
643 wr.flush(wr.buffer, wr.buffer_pos); 656 ret = 0;
644 ret = 0; 657exit_3:
645 large_free(p); 658 large_free(p);
646exit_2: 659exit_2:
647 if (!output) 660 if (!output)
@@ -659,9 +672,9 @@ STATIC int INIT decompress(unsigned char *buf, int in_len,
659 int(*flush)(void*, unsigned int), 672 int(*flush)(void*, unsigned int),
660 unsigned char *output, 673 unsigned char *output,
661 int *posp, 674 int *posp,
662 void(*error_fn)(char *x) 675 void(*error)(char *x)
663 ) 676 )
664{ 677{
665 return unlzma(buf, in_len - 4, fill, flush, output, posp, error_fn); 678 return unlzma(buf, in_len - 4, fill, flush, output, posp, error);
666} 679}
667#endif 680#endif
diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c
index bcb3a4bd68ff..5a7a2adf4c4c 100644
--- a/lib/decompress_unlzo.c
+++ b/lib/decompress_unlzo.c
@@ -33,7 +33,6 @@
33#ifdef STATIC 33#ifdef STATIC
34#include "lzo/lzo1x_decompress.c" 34#include "lzo/lzo1x_decompress.c"
35#else 35#else
36#include <linux/slab.h>
37#include <linux/decompress/unlzo.h> 36#include <linux/decompress/unlzo.h>
38#endif 37#endif
39 38
@@ -49,14 +48,25 @@ static const unsigned char lzop_magic[] = {
49 48
50#define LZO_BLOCK_SIZE (256*1024l) 49#define LZO_BLOCK_SIZE (256*1024l)
51#define HEADER_HAS_FILTER 0x00000800L 50#define HEADER_HAS_FILTER 0x00000800L
51#define HEADER_SIZE_MIN (9 + 7 + 4 + 8 + 1 + 4)
52#define HEADER_SIZE_MAX (9 + 7 + 1 + 8 + 8 + 4 + 1 + 255 + 4)
52 53
53STATIC inline int INIT parse_header(u8 *input, u8 *skip) 54STATIC inline int INIT parse_header(u8 *input, int *skip, int in_len)
54{ 55{
55 int l; 56 int l;
56 u8 *parse = input; 57 u8 *parse = input;
58 u8 *end = input + in_len;
57 u8 level = 0; 59 u8 level = 0;
58 u16 version; 60 u16 version;
59 61
62 /*
63 * Check that there's enough input to possibly have a valid header.
64 * Then it is possible to parse several fields until the minimum
65 * size may have been used.
66 */
67 if (in_len < HEADER_SIZE_MIN)
68 return 0;
69
60 /* read magic: 9 first bits */ 70 /* read magic: 9 first bits */
61 for (l = 0; l < 9; l++) { 71 for (l = 0; l < 9; l++) {
62 if (*parse++ != lzop_magic[l]) 72 if (*parse++ != lzop_magic[l])
@@ -74,6 +84,15 @@ STATIC inline int INIT parse_header(u8 *input, u8 *skip)
74 else 84 else
75 parse += 4; /* flags */ 85 parse += 4; /* flags */
76 86
87 /*
88 * At least mode, mtime_low, filename length, and checksum must
89 * be left to be parsed. If also mtime_high is present, it's OK
90 * because the next input buffer check is after reading the
91 * filename length.
92 */
93 if (end - parse < 8 + 1 + 4)
94 return 0;
95
77 /* skip mode and mtime_low */ 96 /* skip mode and mtime_low */
78 parse += 8; 97 parse += 8;
79 if (version >= 0x0940) 98 if (version >= 0x0940)
@@ -81,6 +100,8 @@ STATIC inline int INIT parse_header(u8 *input, u8 *skip)
81 100
82 l = *parse++; 101 l = *parse++;
83 /* don't care about the file name, and skip checksum */ 102 /* don't care about the file name, and skip checksum */
103 if (end - parse < l + 4)
104 return 0;
84 parse += l + 4; 105 parse += l + 4;
85 106
86 *skip = parse - input; 107 *skip = parse - input;
@@ -91,16 +112,15 @@ STATIC inline int INIT unlzo(u8 *input, int in_len,
91 int (*fill) (void *, unsigned int), 112 int (*fill) (void *, unsigned int),
92 int (*flush) (void *, unsigned int), 113 int (*flush) (void *, unsigned int),
93 u8 *output, int *posp, 114 u8 *output, int *posp,
94 void (*error_fn) (char *x)) 115 void (*error) (char *x))
95{ 116{
96 u8 skip = 0, r = 0; 117 u8 r = 0;
118 int skip = 0;
97 u32 src_len, dst_len; 119 u32 src_len, dst_len;
98 size_t tmp; 120 size_t tmp;
99 u8 *in_buf, *in_buf_save, *out_buf; 121 u8 *in_buf, *in_buf_save, *out_buf;
100 int ret = -1; 122 int ret = -1;
101 123
102 set_error_fn(error_fn);
103
104 if (output) { 124 if (output) {
105 out_buf = output; 125 out_buf = output;
106 } else if (!flush) { 126 } else if (!flush) {
@@ -119,8 +139,8 @@ STATIC inline int INIT unlzo(u8 *input, int in_len,
119 goto exit_1; 139 goto exit_1;
120 } else if (input) { 140 } else if (input) {
121 in_buf = input; 141 in_buf = input;
122 } else if (!fill || !posp) { 142 } else if (!fill) {
123 error("NULL input pointer and missing position pointer or fill function"); 143 error("NULL input pointer and missing fill function");
124 goto exit_1; 144 goto exit_1;
125 } else { 145 } else {
126 in_buf = malloc(lzo1x_worst_compress(LZO_BLOCK_SIZE)); 146 in_buf = malloc(lzo1x_worst_compress(LZO_BLOCK_SIZE));
@@ -134,22 +154,47 @@ STATIC inline int INIT unlzo(u8 *input, int in_len,
134 if (posp) 154 if (posp)
135 *posp = 0; 155 *posp = 0;
136 156
137 if (fill) 157 if (fill) {
138 fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE)); 158 /*
159 * Start from in_buf + HEADER_SIZE_MAX to make it possible
160 * to use memcpy() to copy the unused data to the beginning
161 * of the buffer. This way memmove() isn't needed which
162 * is missing from pre-boot environments of most archs.
163 */
164 in_buf += HEADER_SIZE_MAX;
165 in_len = fill(in_buf, HEADER_SIZE_MAX);
166 }
139 167
140 if (!parse_header(input, &skip)) { 168 if (!parse_header(in_buf, &skip, in_len)) {
141 error("invalid header"); 169 error("invalid header");
142 goto exit_2; 170 goto exit_2;
143 } 171 }
144 in_buf += skip; 172 in_buf += skip;
173 in_len -= skip;
174
175 if (fill) {
176 /* Move the unused data to the beginning of the buffer. */
177 memcpy(in_buf_save, in_buf, in_len);
178 in_buf = in_buf_save;
179 }
145 180
146 if (posp) 181 if (posp)
147 *posp = skip; 182 *posp = skip;
148 183
149 for (;;) { 184 for (;;) {
150 /* read uncompressed block size */ 185 /* read uncompressed block size */
186 if (fill && in_len < 4) {
187 skip = fill(in_buf + in_len, 4 - in_len);
188 if (skip > 0)
189 in_len += skip;
190 }
191 if (in_len < 4) {
192 error("file corrupted");
193 goto exit_2;
194 }
151 dst_len = get_unaligned_be32(in_buf); 195 dst_len = get_unaligned_be32(in_buf);
152 in_buf += 4; 196 in_buf += 4;
197 in_len -= 4;
153 198
154 /* exit if last block */ 199 /* exit if last block */
155 if (dst_len == 0) { 200 if (dst_len == 0) {
@@ -164,8 +209,18 @@ STATIC inline int INIT unlzo(u8 *input, int in_len,
164 } 209 }
165 210
166 /* read compressed block size, and skip block checksum info */ 211 /* read compressed block size, and skip block checksum info */
212 if (fill && in_len < 8) {
213 skip = fill(in_buf + in_len, 8 - in_len);
214 if (skip > 0)
215 in_len += skip;
216 }
217 if (in_len < 8) {
218 error("file corrupted");
219 goto exit_2;
220 }
167 src_len = get_unaligned_be32(in_buf); 221 src_len = get_unaligned_be32(in_buf);
168 in_buf += 8; 222 in_buf += 8;
223 in_len -= 8;
169 224
170 if (src_len <= 0 || src_len > dst_len) { 225 if (src_len <= 0 || src_len > dst_len) {
171 error("file corrupted"); 226 error("file corrupted");
@@ -173,6 +228,15 @@ STATIC inline int INIT unlzo(u8 *input, int in_len,
173 } 228 }
174 229
175 /* decompress */ 230 /* decompress */
231 if (fill && in_len < src_len) {
232 skip = fill(in_buf + in_len, src_len - in_len);
233 if (skip > 0)
234 in_len += skip;
235 }
236 if (in_len < src_len) {
237 error("file corrupted");
238 goto exit_2;
239 }
176 tmp = dst_len; 240 tmp = dst_len;
177 241
178 /* When the input data is not compressed at all, 242 /* When the input data is not compressed at all,
@@ -190,17 +254,26 @@ STATIC inline int INIT unlzo(u8 *input, int in_len,
190 } 254 }
191 } 255 }
192 256
193 if (flush) 257 if (flush && flush(out_buf, dst_len) != dst_len)
194 flush(out_buf, dst_len); 258 goto exit_2;
195 if (output) 259 if (output)
196 out_buf += dst_len; 260 out_buf += dst_len;
197 if (posp) 261 if (posp)
198 *posp += src_len + 12; 262 *posp += src_len + 12;
263
264 in_buf += src_len;
265 in_len -= src_len;
199 if (fill) { 266 if (fill) {
267 /*
268 * If there happens to still be unused data left in
269 * in_buf, move it to the beginning of the buffer.
270 * Use a loop to avoid memmove() dependency.
271 */
272 if (in_len > 0)
273 for (skip = 0; skip < in_len; ++skip)
274 in_buf_save[skip] = in_buf[skip];
200 in_buf = in_buf_save; 275 in_buf = in_buf_save;
201 fill(in_buf, lzo1x_worst_compress(LZO_BLOCK_SIZE)); 276 }
202 } else
203 in_buf += src_len;
204 } 277 }
205 278
206 ret = 0; 279 ret = 0;
diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c
new file mode 100644
index 000000000000..9f34eb56854d
--- /dev/null
+++ b/lib/decompress_unxz.c
@@ -0,0 +1,397 @@
1/*
2 * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd
3 *
4 * Author: Lasse Collin <lasse.collin@tukaani.org>
5 *
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
8 */
9
10/*
11 * Important notes about in-place decompression
12 *
13 * At least on x86, the kernel is decompressed in place: the compressed data
14 * is placed to the end of the output buffer, and the decompressor overwrites
15 * most of the compressed data. There must be enough safety margin to
16 * guarantee that the write position is always behind the read position.
17 *
18 * The safety margin for XZ with LZMA2 or BCJ+LZMA2 is calculated below.
19 * Note that the margin with XZ is bigger than with Deflate (gzip)!
20 *
21 * The worst case for in-place decompression is that the beginning of
22 * the file is compressed extremely well, and the rest of the file is
23 * uncompressible. Thus, we must look for worst-case expansion when the
24 * compressor is encoding uncompressible data.
25 *
26 * The structure of the .xz file in case of a compresed kernel is as follows.
27 * Sizes (as bytes) of the fields are in parenthesis.
28 *
29 * Stream Header (12)
30 * Block Header:
31 * Block Header (8-12)
32 * Compressed Data (N)
33 * Block Padding (0-3)
34 * CRC32 (4)
35 * Index (8-20)
36 * Stream Footer (12)
37 *
38 * Normally there is exactly one Block, but let's assume that there are
39 * 2-4 Blocks just in case. Because Stream Header and also Block Header
40 * of the first Block don't make the decompressor produce any uncompressed
41 * data, we can ignore them from our calculations. Block Headers of possible
42 * additional Blocks have to be taken into account still. With these
43 * assumptions, it is safe to assume that the total header overhead is
44 * less than 128 bytes.
45 *
46 * Compressed Data contains LZMA2 or BCJ+LZMA2 encoded data. Since BCJ
47 * doesn't change the size of the data, it is enough to calculate the
48 * safety margin for LZMA2.
49 *
50 * LZMA2 stores the data in chunks. Each chunk has a header whose size is
51 * a maximum of 6 bytes, but to get round 2^n numbers, let's assume that
52 * the maximum chunk header size is 8 bytes. After the chunk header, there
53 * may be up to 64 KiB of actual payload in the chunk. Often the payload is
54 * quite a bit smaller though; to be safe, let's assume that an average
55 * chunk has only 32 KiB of payload.
56 *
57 * The maximum uncompressed size of the payload is 2 MiB. The minimum
58 * uncompressed size of the payload is in practice never less than the
59 * payload size itself. The LZMA2 format would allow uncompressed size
60 * to be less than the payload size, but no sane compressor creates such
61 * files. LZMA2 supports storing uncompressible data in uncompressed form,
62 * so there's never a need to create payloads whose uncompressed size is
63 * smaller than the compressed size.
64 *
65 * The assumption, that the uncompressed size of the payload is never
66 * smaller than the payload itself, is valid only when talking about
67 * the payload as a whole. It is possible that the payload has parts where
68 * the decompressor consumes more input than it produces output. Calculating
69 * the worst case for this would be tricky. Instead of trying to do that,
70 * let's simply make sure that the decompressor never overwrites any bytes
71 * of the payload which it is currently reading.
72 *
73 * Now we have enough information to calculate the safety margin. We need
74 * - 128 bytes for the .xz file format headers;
75 * - 8 bytes per every 32 KiB of uncompressed size (one LZMA2 chunk header
76 * per chunk, each chunk having average payload size of 32 KiB); and
77 * - 64 KiB (biggest possible LZMA2 chunk payload size) to make sure that
78 * the decompressor never overwrites anything from the LZMA2 chunk
79 * payload it is currently reading.
80 *
81 * We get the following formula:
82 *
83 * safety_margin = 128 + uncompressed_size * 8 / 32768 + 65536
84 * = 128 + (uncompressed_size >> 12) + 65536
85 *
86 * For comparison, according to arch/x86/boot/compressed/misc.c, the
87 * equivalent formula for Deflate is this:
88 *
89 * safety_margin = 18 + (uncompressed_size >> 12) + 32768
90 *
91 * Thus, when updating Deflate-only in-place kernel decompressor to
92 * support XZ, the fixed overhead has to be increased from 18+32768 bytes
93 * to 128+65536 bytes.
94 */
95
96/*
97 * STATIC is defined to "static" if we are being built for kernel
98 * decompression (pre-boot code). <linux/decompress/mm.h> will define
99 * STATIC to empty if it wasn't already defined. Since we will need to
100 * know later if we are being used for kernel decompression, we define
101 * XZ_PREBOOT here.
102 */
103#ifdef STATIC
104# define XZ_PREBOOT
105#endif
106#ifdef __KERNEL__
107# include <linux/decompress/mm.h>
108#endif
109#define XZ_EXTERN STATIC
110
111#ifndef XZ_PREBOOT
112# include <linux/slab.h>
113# include <linux/xz.h>
114#else
115/*
116 * Use the internal CRC32 code instead of kernel's CRC32 module, which
117 * is not available in early phase of booting.
118 */
119#define XZ_INTERNAL_CRC32 1
120
121/*
122 * For boot time use, we enable only the BCJ filter of the current
123 * architecture or none if no BCJ filter is available for the architecture.
124 */
125#ifdef CONFIG_X86
126# define XZ_DEC_X86
127#endif
128#ifdef CONFIG_PPC
129# define XZ_DEC_POWERPC
130#endif
131#ifdef CONFIG_ARM
132# define XZ_DEC_ARM
133#endif
134#ifdef CONFIG_IA64
135# define XZ_DEC_IA64
136#endif
137#ifdef CONFIG_SPARC
138# define XZ_DEC_SPARC
139#endif
140
141/*
142 * This will get the basic headers so that memeq() and others
143 * can be defined.
144 */
145#include "xz/xz_private.h"
146
147/*
148 * Replace the normal allocation functions with the versions from
149 * <linux/decompress/mm.h>. vfree() needs to support vfree(NULL)
150 * when XZ_DYNALLOC is used, but the pre-boot free() doesn't support it.
151 * Workaround it here because the other decompressors don't need it.
152 */
153#undef kmalloc
154#undef kfree
155#undef vmalloc
156#undef vfree
157#define kmalloc(size, flags) malloc(size)
158#define kfree(ptr) free(ptr)
159#define vmalloc(size) malloc(size)
160#define vfree(ptr) do { if (ptr != NULL) free(ptr); } while (0)
161
162/*
163 * FIXME: Not all basic memory functions are provided in architecture-specific
164 * files (yet). We define our own versions here for now, but this should be
165 * only a temporary solution.
166 *
167 * memeq and memzero are not used much and any remotely sane implementation
168 * is fast enough. memcpy/memmove speed matters in multi-call mode, but
169 * the kernel image is decompressed in single-call mode, in which only
170 * memcpy speed can matter and only if there is a lot of uncompressible data
171 * (LZMA2 stores uncompressible chunks in uncompressed form). Thus, the
172 * functions below should just be kept small; it's probably not worth
173 * optimizing for speed.
174 */
175
176#ifndef memeq
177static bool memeq(const void *a, const void *b, size_t size)
178{
179 const uint8_t *x = a;
180 const uint8_t *y = b;
181 size_t i;
182
183 for (i = 0; i < size; ++i)
184 if (x[i] != y[i])
185 return false;
186
187 return true;
188}
189#endif
190
191#ifndef memzero
192static void memzero(void *buf, size_t size)
193{
194 uint8_t *b = buf;
195 uint8_t *e = b + size;
196
197 while (b != e)
198 *b++ = '\0';
199}
200#endif
201
202#ifndef memmove
203/* Not static to avoid a conflict with the prototype in the Linux headers. */
204void *memmove(void *dest, const void *src, size_t size)
205{
206 uint8_t *d = dest;
207 const uint8_t *s = src;
208 size_t i;
209
210 if (d < s) {
211 for (i = 0; i < size; ++i)
212 d[i] = s[i];
213 } else if (d > s) {
214 i = size;
215 while (i-- > 0)
216 d[i] = s[i];
217 }
218
219 return dest;
220}
221#endif
222
223/*
224 * Since we need memmove anyway, would use it as memcpy too.
225 * Commented out for now to avoid breaking things.
226 */
227/*
228#ifndef memcpy
229# define memcpy memmove
230#endif
231*/
232
233#include "xz/xz_crc32.c"
234#include "xz/xz_dec_stream.c"
235#include "xz/xz_dec_lzma2.c"
236#include "xz/xz_dec_bcj.c"
237
238#endif /* XZ_PREBOOT */
239
240/* Size of the input and output buffers in multi-call mode */
241#define XZ_IOBUF_SIZE 4096
242
243/*
244 * This function implements the API defined in <linux/decompress/generic.h>.
245 *
246 * This wrapper will automatically choose single-call or multi-call mode
247 * of the native XZ decoder API. The single-call mode can be used only when
248 * both input and output buffers are available as a single chunk, i.e. when
249 * fill() and flush() won't be used.
250 */
251STATIC int INIT unxz(unsigned char *in, int in_size,
252 int (*fill)(void *dest, unsigned int size),
253 int (*flush)(void *src, unsigned int size),
254 unsigned char *out, int *in_used,
255 void (*error)(char *x))
256{
257 struct xz_buf b;
258 struct xz_dec *s;
259 enum xz_ret ret;
260 bool must_free_in = false;
261
262#if XZ_INTERNAL_CRC32
263 xz_crc32_init();
264#endif
265
266 if (in_used != NULL)
267 *in_used = 0;
268
269 if (fill == NULL && flush == NULL)
270 s = xz_dec_init(XZ_SINGLE, 0);
271 else
272 s = xz_dec_init(XZ_DYNALLOC, (uint32_t)-1);
273
274 if (s == NULL)
275 goto error_alloc_state;
276
277 if (flush == NULL) {
278 b.out = out;
279 b.out_size = (size_t)-1;
280 } else {
281 b.out_size = XZ_IOBUF_SIZE;
282 b.out = malloc(XZ_IOBUF_SIZE);
283 if (b.out == NULL)
284 goto error_alloc_out;
285 }
286
287 if (in == NULL) {
288 must_free_in = true;
289 in = malloc(XZ_IOBUF_SIZE);
290 if (in == NULL)
291 goto error_alloc_in;
292 }
293
294 b.in = in;
295 b.in_pos = 0;
296 b.in_size = in_size;
297 b.out_pos = 0;
298
299 if (fill == NULL && flush == NULL) {
300 ret = xz_dec_run(s, &b);
301 } else {
302 do {
303 if (b.in_pos == b.in_size && fill != NULL) {
304 if (in_used != NULL)
305 *in_used += b.in_pos;
306
307 b.in_pos = 0;
308
309 in_size = fill(in, XZ_IOBUF_SIZE);
310 if (in_size < 0) {
311 /*
312 * This isn't an optimal error code
313 * but it probably isn't worth making
314 * a new one either.
315 */
316 ret = XZ_BUF_ERROR;
317 break;
318 }
319
320 b.in_size = in_size;
321 }
322
323 ret = xz_dec_run(s, &b);
324
325 if (flush != NULL && (b.out_pos == b.out_size
326 || (ret != XZ_OK && b.out_pos > 0))) {
327 /*
328 * Setting ret here may hide an error
329 * returned by xz_dec_run(), but probably
330 * it's not too bad.
331 */
332 if (flush(b.out, b.out_pos) != (int)b.out_pos)
333 ret = XZ_BUF_ERROR;
334
335 b.out_pos = 0;
336 }
337 } while (ret == XZ_OK);
338
339 if (must_free_in)
340 free(in);
341
342 if (flush != NULL)
343 free(b.out);
344 }
345
346 if (in_used != NULL)
347 *in_used += b.in_pos;
348
349 xz_dec_end(s);
350
351 switch (ret) {
352 case XZ_STREAM_END:
353 return 0;
354
355 case XZ_MEM_ERROR:
356 /* This can occur only in multi-call mode. */
357 error("XZ decompressor ran out of memory");
358 break;
359
360 case XZ_FORMAT_ERROR:
361 error("Input is not in the XZ format (wrong magic bytes)");
362 break;
363
364 case XZ_OPTIONS_ERROR:
365 error("Input was encoded with settings that are not "
366 "supported by this XZ decoder");
367 break;
368
369 case XZ_DATA_ERROR:
370 case XZ_BUF_ERROR:
371 error("XZ-compressed data is corrupt");
372 break;
373
374 default:
375 error("Bug in the XZ decompressor");
376 break;
377 }
378
379 return -1;
380
381error_alloc_in:
382 if (flush != NULL)
383 free(b.out);
384
385error_alloc_out:
386 xz_dec_end(s);
387
388error_alloc_state:
389 error("XZ decompressor ran out of memory");
390 return -1;
391}
392
393/*
394 * This macro is used by architecture-specific files to decompress
395 * the kernel image.
396 */
397#define decompress unxz
diff --git a/lib/div64.c b/lib/div64.c
index a111eb8de9cf..5b4919191778 100644
--- a/lib/div64.c
+++ b/lib/div64.c
@@ -77,26 +77,58 @@ s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder)
77EXPORT_SYMBOL(div_s64_rem); 77EXPORT_SYMBOL(div_s64_rem);
78#endif 78#endif
79 79
80/* 64bit divisor, dividend and result. dynamic precision */ 80/**
81 * div64_u64 - unsigned 64bit divide with 64bit divisor
82 * @dividend: 64bit dividend
83 * @divisor: 64bit divisor
84 *
85 * This implementation is a modified version of the algorithm proposed
86 * by the book 'Hacker's Delight'. The original source and full proof
87 * can be found here and is available for use without restriction.
88 *
89 * 'http://www.hackersdelight.org/HDcode/newCode/divDouble.c'
90 */
81#ifndef div64_u64 91#ifndef div64_u64
82u64 div64_u64(u64 dividend, u64 divisor) 92u64 div64_u64(u64 dividend, u64 divisor)
83{ 93{
84 u32 high, d; 94 u32 high = divisor >> 32;
95 u64 quot;
85 96
86 high = divisor >> 32; 97 if (high == 0) {
87 if (high) { 98 quot = div_u64(dividend, divisor);
88 unsigned int shift = fls(high); 99 } else {
100 int n = 1 + fls(high);
101 quot = div_u64(dividend >> n, divisor >> n);
89 102
90 d = divisor >> shift; 103 if (quot != 0)
91 dividend >>= shift; 104 quot--;
92 } else 105 if ((dividend - quot * divisor) >= divisor)
93 d = divisor; 106 quot++;
107 }
94 108
95 return div_u64(dividend, d); 109 return quot;
96} 110}
97EXPORT_SYMBOL(div64_u64); 111EXPORT_SYMBOL(div64_u64);
98#endif 112#endif
99 113
114/**
115 * div64_s64 - signed 64bit divide with 64bit divisor
116 * @dividend: 64bit dividend
117 * @divisor: 64bit divisor
118 */
119#ifndef div64_s64
120s64 div64_s64(s64 dividend, s64 divisor)
121{
122 s64 quot, t;
123
124 quot = div64_u64(abs64(dividend), abs64(divisor));
125 t = (dividend ^ divisor) >> 63;
126
127 return (quot ^ t) - t;
128}
129EXPORT_SYMBOL(div64_s64);
130#endif
131
100#endif /* BITS_PER_LONG == 32 */ 132#endif /* BITS_PER_LONG == 32 */
101 133
102/* 134/*
diff --git a/lib/dma-debug.c b/lib/dma-debug.c
index 01e64270e246..db07bfd9298e 100644
--- a/lib/dma-debug.c
+++ b/lib/dma-debug.c
@@ -590,6 +590,7 @@ out_unlock:
590static const struct file_operations filter_fops = { 590static const struct file_operations filter_fops = {
591 .read = filter_read, 591 .read = filter_read,
592 .write = filter_write, 592 .write = filter_write,
593 .llseek = default_llseek,
593}; 594};
594 595
595static int dma_debug_fs_init(void) 596static int dma_debug_fs_init(void)
@@ -648,7 +649,7 @@ out_err:
648 return -ENOMEM; 649 return -ENOMEM;
649} 650}
650 651
651static int device_dma_allocations(struct device *dev) 652static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
652{ 653{
653 struct dma_debug_entry *entry; 654 struct dma_debug_entry *entry;
654 unsigned long flags; 655 unsigned long flags;
@@ -659,8 +660,10 @@ static int device_dma_allocations(struct device *dev)
659 for (i = 0; i < HASH_SIZE; ++i) { 660 for (i = 0; i < HASH_SIZE; ++i) {
660 spin_lock(&dma_entry_hash[i].lock); 661 spin_lock(&dma_entry_hash[i].lock);
661 list_for_each_entry(entry, &dma_entry_hash[i].list, list) { 662 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
662 if (entry->dev == dev) 663 if (entry->dev == dev) {
663 count += 1; 664 count += 1;
665 *out_entry = entry;
666 }
664 } 667 }
665 spin_unlock(&dma_entry_hash[i].lock); 668 spin_unlock(&dma_entry_hash[i].lock);
666 } 669 }
@@ -673,6 +676,7 @@ static int device_dma_allocations(struct device *dev)
673static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data) 676static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
674{ 677{
675 struct device *dev = data; 678 struct device *dev = data;
679 struct dma_debug_entry *uninitialized_var(entry);
676 int count; 680 int count;
677 681
678 if (global_disable) 682 if (global_disable)
@@ -680,12 +684,17 @@ static int dma_debug_device_change(struct notifier_block *nb, unsigned long acti
680 684
681 switch (action) { 685 switch (action) {
682 case BUS_NOTIFY_UNBOUND_DRIVER: 686 case BUS_NOTIFY_UNBOUND_DRIVER:
683 count = device_dma_allocations(dev); 687 count = device_dma_allocations(dev, &entry);
684 if (count == 0) 688 if (count == 0)
685 break; 689 break;
686 err_printk(dev, NULL, "DMA-API: device driver has pending " 690 err_printk(dev, entry, "DMA-API: device driver has pending "
687 "DMA allocations while released from device " 691 "DMA allocations while released from device "
688 "[count=%d]\n", count); 692 "[count=%d]\n"
693 "One of leaked entries details: "
694 "[device address=0x%016llx] [size=%llu bytes] "
695 "[mapped with %s] [mapped as %s]\n",
696 count, entry->dev_addr, entry->size,
697 dir2name[entry->direction], type2name[entry->type]);
689 break; 698 break;
690 default: 699 default:
691 break; 700 break;
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index 02afc2533728..75ca78f3a8c9 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -7,6 +7,7 @@
7 * Copyright (C) 2008 Jason Baron <jbaron@redhat.com> 7 * Copyright (C) 2008 Jason Baron <jbaron@redhat.com>
8 * By Greg Banks <gnb@melbourne.sgi.com> 8 * By Greg Banks <gnb@melbourne.sgi.com>
9 * Copyright (c) 2008 Silicon Graphics Inc. All Rights Reserved. 9 * Copyright (c) 2008 Silicon Graphics Inc. All Rights Reserved.
10 * Copyright (C) 2011 Bart Van Assche. All Rights Reserved.
10 */ 11 */
11 12
12#include <linux/kernel.h> 13#include <linux/kernel.h>
@@ -26,19 +27,13 @@
26#include <linux/dynamic_debug.h> 27#include <linux/dynamic_debug.h>
27#include <linux/debugfs.h> 28#include <linux/debugfs.h>
28#include <linux/slab.h> 29#include <linux/slab.h>
30#include <linux/jump_label.h>
31#include <linux/hardirq.h>
32#include <linux/sched.h>
29 33
30extern struct _ddebug __start___verbose[]; 34extern struct _ddebug __start___verbose[];
31extern struct _ddebug __stop___verbose[]; 35extern struct _ddebug __stop___verbose[];
32 36
33/* dynamic_debug_enabled, and dynamic_debug_enabled2 are bitmasks in which
34 * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
35 * use independent hash functions, to reduce the chance of false positives.
36 */
37long long dynamic_debug_enabled;
38EXPORT_SYMBOL_GPL(dynamic_debug_enabled);
39long long dynamic_debug_enabled2;
40EXPORT_SYMBOL_GPL(dynamic_debug_enabled2);
41
42struct ddebug_table { 37struct ddebug_table {
43 struct list_head link; 38 struct list_head link;
44 char *mod_name; 39 char *mod_name;
@@ -71,15 +66,25 @@ static inline const char *basename(const char *path)
71 return tail ? tail+1 : path; 66 return tail ? tail+1 : path;
72} 67}
73 68
69static struct { unsigned flag:8; char opt_char; } opt_array[] = {
70 { _DPRINTK_FLAGS_PRINT, 'p' },
71 { _DPRINTK_FLAGS_INCL_MODNAME, 'm' },
72 { _DPRINTK_FLAGS_INCL_FUNCNAME, 'f' },
73 { _DPRINTK_FLAGS_INCL_LINENO, 'l' },
74 { _DPRINTK_FLAGS_INCL_TID, 't' },
75};
76
74/* format a string into buf[] which describes the _ddebug's flags */ 77/* format a string into buf[] which describes the _ddebug's flags */
75static char *ddebug_describe_flags(struct _ddebug *dp, char *buf, 78static char *ddebug_describe_flags(struct _ddebug *dp, char *buf,
76 size_t maxlen) 79 size_t maxlen)
77{ 80{
78 char *p = buf; 81 char *p = buf;
82 int i;
79 83
80 BUG_ON(maxlen < 4); 84 BUG_ON(maxlen < 4);
81 if (dp->flags & _DPRINTK_FLAGS_PRINT) 85 for (i = 0; i < ARRAY_SIZE(opt_array); ++i)
82 *p++ = 'p'; 86 if (dp->flags & opt_array[i].flag)
87 *p++ = opt_array[i].opt_char;
83 if (p == buf) 88 if (p == buf)
84 *p++ = '-'; 89 *p++ = '-';
85 *p = '\0'; 90 *p = '\0';
@@ -88,26 +93,6 @@ static char *ddebug_describe_flags(struct _ddebug *dp, char *buf,
88} 93}
89 94
90/* 95/*
91 * must be called with ddebug_lock held
92 */
93
94static int disabled_hash(char hash, bool first_table)
95{
96 struct ddebug_table *dt;
97 char table_hash_value;
98
99 list_for_each_entry(dt, &ddebug_tables, link) {
100 if (first_table)
101 table_hash_value = dt->ddebugs->primary_hash;
102 else
103 table_hash_value = dt->ddebugs->secondary_hash;
104 if (dt->num_enabled && (hash == table_hash_value))
105 return 0;
106 }
107 return 1;
108}
109
110/*
111 * Search the tables for _ddebug's which match the given 96 * Search the tables for _ddebug's which match the given
112 * `query' and apply the `flags' and `mask' to them. Tells 97 * `query' and apply the `flags' and `mask' to them. Tells
113 * the user which ddebug's were changed, or whether none 98 * the user which ddebug's were changed, or whether none
@@ -169,19 +154,10 @@ static void ddebug_change(const struct ddebug_query *query,
169 else if (!dp->flags) 154 else if (!dp->flags)
170 dt->num_enabled++; 155 dt->num_enabled++;
171 dp->flags = newflags; 156 dp->flags = newflags;
172 if (newflags) { 157 if (newflags)
173 dynamic_debug_enabled |= 158 dp->enabled = 1;
174 (1LL << dp->primary_hash); 159 else
175 dynamic_debug_enabled2 |= 160 dp->enabled = 0;
176 (1LL << dp->secondary_hash);
177 } else {
178 if (disabled_hash(dp->primary_hash, true))
179 dynamic_debug_enabled &=
180 ~(1LL << dp->primary_hash);
181 if (disabled_hash(dp->secondary_hash, false))
182 dynamic_debug_enabled2 &=
183 ~(1LL << dp->secondary_hash);
184 }
185 if (verbose) 161 if (verbose)
186 printk(KERN_INFO 162 printk(KERN_INFO
187 "ddebug: changed %s:%d [%s]%s %s\n", 163 "ddebug: changed %s:%d [%s]%s %s\n",
@@ -380,7 +356,7 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
380 unsigned int *maskp) 356 unsigned int *maskp)
381{ 357{
382 unsigned flags = 0; 358 unsigned flags = 0;
383 int op = '='; 359 int op = '=', i;
384 360
385 switch (*str) { 361 switch (*str) {
386 case '+': 362 case '+':
@@ -395,13 +371,14 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
395 printk(KERN_INFO "%s: op='%c'\n", __func__, op); 371 printk(KERN_INFO "%s: op='%c'\n", __func__, op);
396 372
397 for ( ; *str ; ++str) { 373 for ( ; *str ; ++str) {
398 switch (*str) { 374 for (i = ARRAY_SIZE(opt_array) - 1; i >= 0; i--) {
399 case 'p': 375 if (*str == opt_array[i].opt_char) {
400 flags |= _DPRINTK_FLAGS_PRINT; 376 flags |= opt_array[i].flag;
401 break; 377 break;
402 default: 378 }
403 return -EINVAL;
404 } 379 }
380 if (i < 0)
381 return -EINVAL;
405 } 382 }
406 if (flags == 0) 383 if (flags == 0)
407 return -EINVAL; 384 return -EINVAL;
@@ -429,6 +406,69 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
429 return 0; 406 return 0;
430} 407}
431 408
409static int ddebug_exec_query(char *query_string)
410{
411 unsigned int flags = 0, mask = 0;
412 struct ddebug_query query;
413#define MAXWORDS 9
414 int nwords;
415 char *words[MAXWORDS];
416
417 nwords = ddebug_tokenize(query_string, words, MAXWORDS);
418 if (nwords <= 0)
419 return -EINVAL;
420 if (ddebug_parse_query(words, nwords-1, &query))
421 return -EINVAL;
422 if (ddebug_parse_flags(words[nwords-1], &flags, &mask))
423 return -EINVAL;
424
425 /* actually go and implement the change */
426 ddebug_change(&query, flags, mask);
427 return 0;
428}
429
430int __dynamic_pr_debug(struct _ddebug *descriptor, const char *fmt, ...)
431{
432 va_list args;
433 int res;
434
435 BUG_ON(!descriptor);
436 BUG_ON(!fmt);
437
438 va_start(args, fmt);
439 res = printk(KERN_DEBUG);
440 if (descriptor->flags & _DPRINTK_FLAGS_INCL_TID) {
441 if (in_interrupt())
442 res += printk(KERN_CONT "<intr> ");
443 else
444 res += printk(KERN_CONT "[%d] ", task_pid_vnr(current));
445 }
446 if (descriptor->flags & _DPRINTK_FLAGS_INCL_MODNAME)
447 res += printk(KERN_CONT "%s:", descriptor->modname);
448 if (descriptor->flags & _DPRINTK_FLAGS_INCL_FUNCNAME)
449 res += printk(KERN_CONT "%s:", descriptor->function);
450 if (descriptor->flags & _DPRINTK_FLAGS_INCL_LINENO)
451 res += printk(KERN_CONT "%d ", descriptor->lineno);
452 res += vprintk(fmt, args);
453 va_end(args);
454
455 return res;
456}
457EXPORT_SYMBOL(__dynamic_pr_debug);
458
459static __initdata char ddebug_setup_string[1024];
460static __init int ddebug_setup_query(char *str)
461{
462 if (strlen(str) >= 1024) {
463 pr_warning("ddebug boot param string too large\n");
464 return 0;
465 }
466 strcpy(ddebug_setup_string, str);
467 return 1;
468}
469
470__setup("ddebug_query=", ddebug_setup_query);
471
432/* 472/*
433 * File_ops->write method for <debugfs>/dynamic_debug/conrol. Gathers the 473 * File_ops->write method for <debugfs>/dynamic_debug/conrol. Gathers the
434 * command text from userspace, parses and executes it. 474 * command text from userspace, parses and executes it.
@@ -436,12 +476,8 @@ static int ddebug_parse_flags(const char *str, unsigned int *flagsp,
436static ssize_t ddebug_proc_write(struct file *file, const char __user *ubuf, 476static ssize_t ddebug_proc_write(struct file *file, const char __user *ubuf,
437 size_t len, loff_t *offp) 477 size_t len, loff_t *offp)
438{ 478{
439 unsigned int flags = 0, mask = 0;
440 struct ddebug_query query;
441#define MAXWORDS 9
442 int nwords;
443 char *words[MAXWORDS];
444 char tmpbuf[256]; 479 char tmpbuf[256];
480 int ret;
445 481
446 if (len == 0) 482 if (len == 0)
447 return 0; 483 return 0;
@@ -455,16 +491,9 @@ static ssize_t ddebug_proc_write(struct file *file, const char __user *ubuf,
455 printk(KERN_INFO "%s: read %d bytes from userspace\n", 491 printk(KERN_INFO "%s: read %d bytes from userspace\n",
456 __func__, (int)len); 492 __func__, (int)len);
457 493
458 nwords = ddebug_tokenize(tmpbuf, words, MAXWORDS); 494 ret = ddebug_exec_query(tmpbuf);
459 if (nwords <= 0) 495 if (ret)
460 return -EINVAL; 496 return ret;
461 if (ddebug_parse_query(words, nwords-1, &query))
462 return -EINVAL;
463 if (ddebug_parse_flags(words[nwords-1], &flags, &mask))
464 return -EINVAL;
465
466 /* actually go and implement the change */
467 ddebug_change(&query, flags, mask);
468 497
469 *offp += len; 498 *offp += len;
470 return len; 499 return len;
@@ -725,13 +754,14 @@ static void ddebug_remove_all_tables(void)
725 mutex_unlock(&ddebug_lock); 754 mutex_unlock(&ddebug_lock);
726} 755}
727 756
728static int __init dynamic_debug_init(void) 757static __initdata int ddebug_init_success;
758
759static int __init dynamic_debug_init_debugfs(void)
729{ 760{
730 struct dentry *dir, *file; 761 struct dentry *dir, *file;
731 struct _ddebug *iter, *iter_start; 762
732 const char *modname = NULL; 763 if (!ddebug_init_success)
733 int ret = 0; 764 return -ENODEV;
734 int n = 0;
735 765
736 dir = debugfs_create_dir("dynamic_debug", NULL); 766 dir = debugfs_create_dir("dynamic_debug", NULL);
737 if (!dir) 767 if (!dir)
@@ -742,6 +772,16 @@ static int __init dynamic_debug_init(void)
742 debugfs_remove(dir); 772 debugfs_remove(dir);
743 return -ENOMEM; 773 return -ENOMEM;
744 } 774 }
775 return 0;
776}
777
778static int __init dynamic_debug_init(void)
779{
780 struct _ddebug *iter, *iter_start;
781 const char *modname = NULL;
782 int ret = 0;
783 int n = 0;
784
745 if (__start___verbose != __stop___verbose) { 785 if (__start___verbose != __stop___verbose) {
746 iter = __start___verbose; 786 iter = __start___verbose;
747 modname = iter->modname; 787 modname = iter->modname;
@@ -759,12 +799,26 @@ static int __init dynamic_debug_init(void)
759 } 799 }
760 ret = ddebug_add_module(iter_start, n, modname); 800 ret = ddebug_add_module(iter_start, n, modname);
761 } 801 }
802
803 /* ddebug_query boot param got passed -> set it up */
804 if (ddebug_setup_string[0] != '\0') {
805 ret = ddebug_exec_query(ddebug_setup_string);
806 if (ret)
807 pr_warning("Invalid ddebug boot param %s",
808 ddebug_setup_string);
809 else
810 pr_info("ddebug initialized with string %s",
811 ddebug_setup_string);
812 }
813
762out_free: 814out_free:
763 if (ret) { 815 if (ret)
764 ddebug_remove_all_tables(); 816 ddebug_remove_all_tables();
765 debugfs_remove(dir); 817 else
766 debugfs_remove(file); 818 ddebug_init_success = 1;
767 }
768 return 0; 819 return 0;
769} 820}
770module_init(dynamic_debug_init); 821/* Allow early initialization for boot messages via boot param */
822arch_initcall(dynamic_debug_init);
823/* Debugfs setup must be done later */
824module_init(dynamic_debug_init_debugfs);
diff --git a/lib/find_last_bit.c b/lib/find_last_bit.c
index 5d202e36bdd8..d903959ad695 100644
--- a/lib/find_last_bit.c
+++ b/lib/find_last_bit.c
@@ -15,6 +15,8 @@
15#include <asm/types.h> 15#include <asm/types.h>
16#include <asm/byteorder.h> 16#include <asm/byteorder.h>
17 17
18#ifndef find_last_bit
19
18unsigned long find_last_bit(const unsigned long *addr, unsigned long size) 20unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
19{ 21{
20 unsigned long words; 22 unsigned long words;
@@ -43,3 +45,5 @@ found:
43 return size; 45 return size;
44} 46}
45EXPORT_SYMBOL(find_last_bit); 47EXPORT_SYMBOL(find_last_bit);
48
49#endif
diff --git a/lib/find_next_bit.c b/lib/find_next_bit.c
index 24c59ded47a0..4bd75a73ba00 100644
--- a/lib/find_next_bit.c
+++ b/lib/find_next_bit.c
@@ -16,7 +16,7 @@
16 16
17#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG) 17#define BITOP_WORD(nr) ((nr) / BITS_PER_LONG)
18 18
19#ifdef CONFIG_GENERIC_FIND_NEXT_BIT 19#ifndef find_next_bit
20/* 20/*
21 * Find the next set bit in a memory region. 21 * Find the next set bit in a memory region.
22 */ 22 */
@@ -59,7 +59,9 @@ found_middle:
59 return result + __ffs(tmp); 59 return result + __ffs(tmp);
60} 60}
61EXPORT_SYMBOL(find_next_bit); 61EXPORT_SYMBOL(find_next_bit);
62#endif
62 63
64#ifndef find_next_zero_bit
63/* 65/*
64 * This implementation of find_{first,next}_zero_bit was stolen from 66 * This implementation of find_{first,next}_zero_bit was stolen from
65 * Linus' asm-alpha/bitops.h. 67 * Linus' asm-alpha/bitops.h.
@@ -103,9 +105,9 @@ found_middle:
103 return result + ffz(tmp); 105 return result + ffz(tmp);
104} 106}
105EXPORT_SYMBOL(find_next_zero_bit); 107EXPORT_SYMBOL(find_next_zero_bit);
106#endif /* CONFIG_GENERIC_FIND_NEXT_BIT */ 108#endif
107 109
108#ifdef CONFIG_GENERIC_FIND_FIRST_BIT 110#ifndef find_first_bit
109/* 111/*
110 * Find the first set bit in a memory region. 112 * Find the first set bit in a memory region.
111 */ 113 */
@@ -131,7 +133,9 @@ found:
131 return result + __ffs(tmp); 133 return result + __ffs(tmp);
132} 134}
133EXPORT_SYMBOL(find_first_bit); 135EXPORT_SYMBOL(find_first_bit);
136#endif
134 137
138#ifndef find_first_zero_bit
135/* 139/*
136 * Find the first cleared bit in a memory region. 140 * Find the first cleared bit in a memory region.
137 */ 141 */
@@ -157,7 +161,7 @@ found:
157 return result + ffz(tmp); 161 return result + ffz(tmp);
158} 162}
159EXPORT_SYMBOL(find_first_zero_bit); 163EXPORT_SYMBOL(find_first_zero_bit);
160#endif /* CONFIG_GENERIC_FIND_FIRST_BIT */ 164#endif
161 165
162#ifdef __BIG_ENDIAN 166#ifdef __BIG_ENDIAN
163 167
@@ -185,15 +189,17 @@ static inline unsigned long ext2_swab(const unsigned long y)
185#endif 189#endif
186} 190}
187 191
188unsigned long generic_find_next_zero_le_bit(const unsigned long *addr, unsigned 192#ifndef find_next_zero_bit_le
193unsigned long find_next_zero_bit_le(const void *addr, unsigned
189 long size, unsigned long offset) 194 long size, unsigned long offset)
190{ 195{
191 const unsigned long *p = addr + BITOP_WORD(offset); 196 const unsigned long *p = addr;
192 unsigned long result = offset & ~(BITS_PER_LONG - 1); 197 unsigned long result = offset & ~(BITS_PER_LONG - 1);
193 unsigned long tmp; 198 unsigned long tmp;
194 199
195 if (offset >= size) 200 if (offset >= size)
196 return size; 201 return size;
202 p += BITOP_WORD(offset);
197 size -= result; 203 size -= result;
198 offset &= (BITS_PER_LONG - 1UL); 204 offset &= (BITS_PER_LONG - 1UL);
199 if (offset) { 205 if (offset) {
@@ -226,18 +232,20 @@ found_middle:
226found_middle_swap: 232found_middle_swap:
227 return result + ffz(ext2_swab(tmp)); 233 return result + ffz(ext2_swab(tmp));
228} 234}
235EXPORT_SYMBOL(find_next_zero_bit_le);
236#endif
229 237
230EXPORT_SYMBOL(generic_find_next_zero_le_bit); 238#ifndef find_next_bit_le
231 239unsigned long find_next_bit_le(const void *addr, unsigned
232unsigned long generic_find_next_le_bit(const unsigned long *addr, unsigned
233 long size, unsigned long offset) 240 long size, unsigned long offset)
234{ 241{
235 const unsigned long *p = addr + BITOP_WORD(offset); 242 const unsigned long *p = addr;
236 unsigned long result = offset & ~(BITS_PER_LONG - 1); 243 unsigned long result = offset & ~(BITS_PER_LONG - 1);
237 unsigned long tmp; 244 unsigned long tmp;
238 245
239 if (offset >= size) 246 if (offset >= size)
240 return size; 247 return size;
248 p += BITOP_WORD(offset);
241 size -= result; 249 size -= result;
242 offset &= (BITS_PER_LONG - 1UL); 250 offset &= (BITS_PER_LONG - 1UL);
243 if (offset) { 251 if (offset) {
@@ -271,5 +279,7 @@ found_middle:
271found_middle_swap: 279found_middle_swap:
272 return result + __ffs(ext2_swab(tmp)); 280 return result + __ffs(ext2_swab(tmp));
273} 281}
274EXPORT_SYMBOL(generic_find_next_le_bit); 282EXPORT_SYMBOL(find_next_bit_le);
283#endif
284
275#endif /* __BIG_ENDIAN */ 285#endif /* __BIG_ENDIAN */
diff --git a/lib/flex_array.c b/lib/flex_array.c
index 77a6fea7481e..9b8b89458c4c 100644
--- a/lib/flex_array.c
+++ b/lib/flex_array.c
@@ -23,6 +23,8 @@
23#include <linux/flex_array.h> 23#include <linux/flex_array.h>
24#include <linux/slab.h> 24#include <linux/slab.h>
25#include <linux/stddef.h> 25#include <linux/stddef.h>
26#include <linux/module.h>
27#include <linux/reciprocal_div.h>
26 28
27struct flex_array_part { 29struct flex_array_part {
28 char elements[FLEX_ARRAY_PART_SIZE]; 30 char elements[FLEX_ARRAY_PART_SIZE];
@@ -69,15 +71,15 @@ static inline int elements_fit_in_base(struct flex_array *fa)
69 * Element size | Objects | Objects | 71 * Element size | Objects | Objects |
70 * PAGE_SIZE=4k | 32-bit | 64-bit | 72 * PAGE_SIZE=4k | 32-bit | 64-bit |
71 * ---------------------------------| 73 * ---------------------------------|
72 * 1 bytes | 4186112 | 2093056 | 74 * 1 bytes | 4177920 | 2088960 |
73 * 2 bytes | 2093056 | 1046528 | 75 * 2 bytes | 2088960 | 1044480 |
74 * 3 bytes | 1395030 | 697515 | 76 * 3 bytes | 1392300 | 696150 |
75 * 4 bytes | 1046528 | 523264 | 77 * 4 bytes | 1044480 | 522240 |
76 * 32 bytes | 130816 | 65408 | 78 * 32 bytes | 130560 | 65408 |
77 * 33 bytes | 126728 | 63364 | 79 * 33 bytes | 126480 | 63240 |
78 * 2048 bytes | 2044 | 1022 | 80 * 2048 bytes | 2040 | 1020 |
79 * 2049 bytes | 1022 | 511 | 81 * 2049 bytes | 1020 | 510 |
80 * void * | 1046528 | 261632 | 82 * void * | 1044480 | 261120 |
81 * 83 *
82 * Since 64-bit pointers are twice the size, we lose half the 84 * Since 64-bit pointers are twice the size, we lose half the
83 * capacity in the base structure. Also note that no effort is made 85 * capacity in the base structure. Also note that no effort is made
@@ -87,8 +89,15 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total,
87 gfp_t flags) 89 gfp_t flags)
88{ 90{
89 struct flex_array *ret; 91 struct flex_array *ret;
90 int max_size = FLEX_ARRAY_NR_BASE_PTRS * 92 int elems_per_part = 0;
91 FLEX_ARRAY_ELEMENTS_PER_PART(element_size); 93 int reciprocal_elems = 0;
94 int max_size = 0;
95
96 if (element_size) {
97 elems_per_part = FLEX_ARRAY_ELEMENTS_PER_PART(element_size);
98 reciprocal_elems = reciprocal_value(elems_per_part);
99 max_size = FLEX_ARRAY_NR_BASE_PTRS * elems_per_part;
100 }
92 101
93 /* max_size will end up 0 if element_size > PAGE_SIZE */ 102 /* max_size will end up 0 if element_size > PAGE_SIZE */
94 if (total > max_size) 103 if (total > max_size)
@@ -98,16 +107,19 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total,
98 return NULL; 107 return NULL;
99 ret->element_size = element_size; 108 ret->element_size = element_size;
100 ret->total_nr_elements = total; 109 ret->total_nr_elements = total;
110 ret->elems_per_part = elems_per_part;
111 ret->reciprocal_elems = reciprocal_elems;
101 if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) 112 if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO))
102 memset(&ret->parts[0], FLEX_ARRAY_FREE, 113 memset(&ret->parts[0], FLEX_ARRAY_FREE,
103 FLEX_ARRAY_BASE_BYTES_LEFT); 114 FLEX_ARRAY_BASE_BYTES_LEFT);
104 return ret; 115 return ret;
105} 116}
117EXPORT_SYMBOL(flex_array_alloc);
106 118
107static int fa_element_to_part_nr(struct flex_array *fa, 119static int fa_element_to_part_nr(struct flex_array *fa,
108 unsigned int element_nr) 120 unsigned int element_nr)
109{ 121{
110 return element_nr / FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); 122 return reciprocal_divide(element_nr, fa->reciprocal_elems);
111} 123}
112 124
113/** 125/**
@@ -126,20 +138,22 @@ void flex_array_free_parts(struct flex_array *fa)
126 for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) 138 for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++)
127 kfree(fa->parts[part_nr]); 139 kfree(fa->parts[part_nr]);
128} 140}
141EXPORT_SYMBOL(flex_array_free_parts);
129 142
130void flex_array_free(struct flex_array *fa) 143void flex_array_free(struct flex_array *fa)
131{ 144{
132 flex_array_free_parts(fa); 145 flex_array_free_parts(fa);
133 kfree(fa); 146 kfree(fa);
134} 147}
148EXPORT_SYMBOL(flex_array_free);
135 149
136static unsigned int index_inside_part(struct flex_array *fa, 150static unsigned int index_inside_part(struct flex_array *fa,
137 unsigned int element_nr) 151 unsigned int element_nr,
152 unsigned int part_nr)
138{ 153{
139 unsigned int part_offset; 154 unsigned int part_offset;
140 155
141 part_offset = element_nr % 156 part_offset = element_nr - part_nr * fa->elems_per_part;
142 FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size);
143 return part_offset * fa->element_size; 157 return part_offset * fa->element_size;
144} 158}
145 159
@@ -179,23 +193,27 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags)
179int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, 193int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src,
180 gfp_t flags) 194 gfp_t flags)
181{ 195{
182 int part_nr = fa_element_to_part_nr(fa, element_nr); 196 int part_nr = 0;
183 struct flex_array_part *part; 197 struct flex_array_part *part;
184 void *dst; 198 void *dst;
185 199
186 if (element_nr >= fa->total_nr_elements) 200 if (element_nr >= fa->total_nr_elements)
187 return -ENOSPC; 201 return -ENOSPC;
202 if (!fa->element_size)
203 return 0;
188 if (elements_fit_in_base(fa)) 204 if (elements_fit_in_base(fa))
189 part = (struct flex_array_part *)&fa->parts[0]; 205 part = (struct flex_array_part *)&fa->parts[0];
190 else { 206 else {
207 part_nr = fa_element_to_part_nr(fa, element_nr);
191 part = __fa_get_part(fa, part_nr, flags); 208 part = __fa_get_part(fa, part_nr, flags);
192 if (!part) 209 if (!part)
193 return -ENOMEM; 210 return -ENOMEM;
194 } 211 }
195 dst = &part->elements[index_inside_part(fa, element_nr)]; 212 dst = &part->elements[index_inside_part(fa, element_nr, part_nr)];
196 memcpy(dst, src, fa->element_size); 213 memcpy(dst, src, fa->element_size);
197 return 0; 214 return 0;
198} 215}
216EXPORT_SYMBOL(flex_array_put);
199 217
200/** 218/**
201 * flex_array_clear - clear element in array at @element_nr 219 * flex_array_clear - clear element in array at @element_nr
@@ -206,30 +224,34 @@ int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src,
206 */ 224 */
207int flex_array_clear(struct flex_array *fa, unsigned int element_nr) 225int flex_array_clear(struct flex_array *fa, unsigned int element_nr)
208{ 226{
209 int part_nr = fa_element_to_part_nr(fa, element_nr); 227 int part_nr = 0;
210 struct flex_array_part *part; 228 struct flex_array_part *part;
211 void *dst; 229 void *dst;
212 230
213 if (element_nr >= fa->total_nr_elements) 231 if (element_nr >= fa->total_nr_elements)
214 return -ENOSPC; 232 return -ENOSPC;
233 if (!fa->element_size)
234 return 0;
215 if (elements_fit_in_base(fa)) 235 if (elements_fit_in_base(fa))
216 part = (struct flex_array_part *)&fa->parts[0]; 236 part = (struct flex_array_part *)&fa->parts[0];
217 else { 237 else {
238 part_nr = fa_element_to_part_nr(fa, element_nr);
218 part = fa->parts[part_nr]; 239 part = fa->parts[part_nr];
219 if (!part) 240 if (!part)
220 return -EINVAL; 241 return -EINVAL;
221 } 242 }
222 dst = &part->elements[index_inside_part(fa, element_nr)]; 243 dst = &part->elements[index_inside_part(fa, element_nr, part_nr)];
223 memset(dst, FLEX_ARRAY_FREE, fa->element_size); 244 memset(dst, FLEX_ARRAY_FREE, fa->element_size);
224 return 0; 245 return 0;
225} 246}
247EXPORT_SYMBOL(flex_array_clear);
226 248
227/** 249/**
228 * flex_array_prealloc - guarantee that array space exists 250 * flex_array_prealloc - guarantee that array space exists
229 * @fa: the flex array for which to preallocate parts 251 * @fa: the flex array for which to preallocate parts
230 * @start: index of first array element for which space is allocated 252 * @start: index of first array element for which space is allocated
231 * @end: index of last (inclusive) element for which space is allocated 253 * @nr_elements: number of elements for which space is allocated
232 * @flags: page allocation flags 254 * @flags: page allocation flags
233 * 255 *
234 * This will guarantee that no future calls to flex_array_put() 256 * This will guarantee that no future calls to flex_array_put()
235 * will allocate memory. It can be used if you are expecting to 257 * will allocate memory. It can be used if you are expecting to
@@ -239,15 +261,27 @@ int flex_array_clear(struct flex_array *fa, unsigned int element_nr)
239 * Locking must be provided by the caller. 261 * Locking must be provided by the caller.
240 */ 262 */
241int flex_array_prealloc(struct flex_array *fa, unsigned int start, 263int flex_array_prealloc(struct flex_array *fa, unsigned int start,
242 unsigned int end, gfp_t flags) 264 unsigned int nr_elements, gfp_t flags)
243{ 265{
244 int start_part; 266 int start_part;
245 int end_part; 267 int end_part;
246 int part_nr; 268 int part_nr;
269 unsigned int end;
247 struct flex_array_part *part; 270 struct flex_array_part *part;
248 271
249 if (start >= fa->total_nr_elements || end >= fa->total_nr_elements) 272 if (!start && !nr_elements)
273 return 0;
274 if (start >= fa->total_nr_elements)
250 return -ENOSPC; 275 return -ENOSPC;
276 if (!nr_elements)
277 return 0;
278
279 end = start + nr_elements - 1;
280
281 if (end >= fa->total_nr_elements)
282 return -ENOSPC;
283 if (!fa->element_size)
284 return 0;
251 if (elements_fit_in_base(fa)) 285 if (elements_fit_in_base(fa))
252 return 0; 286 return 0;
253 start_part = fa_element_to_part_nr(fa, start); 287 start_part = fa_element_to_part_nr(fa, start);
@@ -259,6 +293,7 @@ int flex_array_prealloc(struct flex_array *fa, unsigned int start,
259 } 293 }
260 return 0; 294 return 0;
261} 295}
296EXPORT_SYMBOL(flex_array_prealloc);
262 297
263/** 298/**
264 * flex_array_get - pull data back out of the array 299 * flex_array_get - pull data back out of the array
@@ -274,20 +309,24 @@ int flex_array_prealloc(struct flex_array *fa, unsigned int start,
274 */ 309 */
275void *flex_array_get(struct flex_array *fa, unsigned int element_nr) 310void *flex_array_get(struct flex_array *fa, unsigned int element_nr)
276{ 311{
277 int part_nr = fa_element_to_part_nr(fa, element_nr); 312 int part_nr = 0;
278 struct flex_array_part *part; 313 struct flex_array_part *part;
279 314
315 if (!fa->element_size)
316 return NULL;
280 if (element_nr >= fa->total_nr_elements) 317 if (element_nr >= fa->total_nr_elements)
281 return NULL; 318 return NULL;
282 if (elements_fit_in_base(fa)) 319 if (elements_fit_in_base(fa))
283 part = (struct flex_array_part *)&fa->parts[0]; 320 part = (struct flex_array_part *)&fa->parts[0];
284 else { 321 else {
322 part_nr = fa_element_to_part_nr(fa, element_nr);
285 part = fa->parts[part_nr]; 323 part = fa->parts[part_nr];
286 if (!part) 324 if (!part)
287 return NULL; 325 return NULL;
288 } 326 }
289 return &part->elements[index_inside_part(fa, element_nr)]; 327 return &part->elements[index_inside_part(fa, element_nr, part_nr)];
290} 328}
329EXPORT_SYMBOL(flex_array_get);
291 330
292/** 331/**
293 * flex_array_get_ptr - pull a ptr back out of the array 332 * flex_array_get_ptr - pull a ptr back out of the array
@@ -308,6 +347,7 @@ void *flex_array_get_ptr(struct flex_array *fa, unsigned int element_nr)
308 347
309 return *tmp; 348 return *tmp;
310} 349}
350EXPORT_SYMBOL(flex_array_get_ptr);
311 351
312static int part_is_free(struct flex_array_part *part) 352static int part_is_free(struct flex_array_part *part)
313{ 353{
@@ -334,6 +374,8 @@ int flex_array_shrink(struct flex_array *fa)
334 int part_nr; 374 int part_nr;
335 int ret = 0; 375 int ret = 0;
336 376
377 if (!fa->total_nr_elements || !fa->element_size)
378 return 0;
337 if (elements_fit_in_base(fa)) 379 if (elements_fit_in_base(fa))
338 return ret; 380 return ret;
339 for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) { 381 for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) {
@@ -348,3 +390,4 @@ int flex_array_shrink(struct flex_array *fa)
348 } 390 }
349 return ret; 391 return ret;
350} 392}
393EXPORT_SYMBOL(flex_array_shrink);
diff --git a/lib/genalloc.c b/lib/genalloc.c
index 1923f1490e72..577ddf805975 100644
--- a/lib/genalloc.c
+++ b/lib/genalloc.c
@@ -39,17 +39,20 @@ struct gen_pool *gen_pool_create(int min_alloc_order, int nid)
39EXPORT_SYMBOL(gen_pool_create); 39EXPORT_SYMBOL(gen_pool_create);
40 40
41/** 41/**
42 * gen_pool_add - add a new chunk of special memory to the pool 42 * gen_pool_add_virt - add a new chunk of special memory to the pool
43 * @pool: pool to add new memory chunk to 43 * @pool: pool to add new memory chunk to
44 * @addr: starting address of memory chunk to add to pool 44 * @virt: virtual starting address of memory chunk to add to pool
45 * @phys: physical starting address of memory chunk to add to pool
45 * @size: size in bytes of the memory chunk to add to pool 46 * @size: size in bytes of the memory chunk to add to pool
46 * @nid: node id of the node the chunk structure and bitmap should be 47 * @nid: node id of the node the chunk structure and bitmap should be
47 * allocated on, or -1 48 * allocated on, or -1
48 * 49 *
49 * Add a new chunk of special memory to the specified pool. 50 * Add a new chunk of special memory to the specified pool.
51 *
52 * Returns 0 on success or a -ve errno on failure.
50 */ 53 */
51int gen_pool_add(struct gen_pool *pool, unsigned long addr, size_t size, 54int gen_pool_add_virt(struct gen_pool *pool, unsigned long virt, phys_addr_t phys,
52 int nid) 55 size_t size, int nid)
53{ 56{
54 struct gen_pool_chunk *chunk; 57 struct gen_pool_chunk *chunk;
55 int nbits = size >> pool->min_alloc_order; 58 int nbits = size >> pool->min_alloc_order;
@@ -58,11 +61,12 @@ int gen_pool_add(struct gen_pool *pool, unsigned long addr, size_t size,
58 61
59 chunk = kmalloc_node(nbytes, GFP_KERNEL | __GFP_ZERO, nid); 62 chunk = kmalloc_node(nbytes, GFP_KERNEL | __GFP_ZERO, nid);
60 if (unlikely(chunk == NULL)) 63 if (unlikely(chunk == NULL))
61 return -1; 64 return -ENOMEM;
62 65
63 spin_lock_init(&chunk->lock); 66 spin_lock_init(&chunk->lock);
64 chunk->start_addr = addr; 67 chunk->phys_addr = phys;
65 chunk->end_addr = addr + size; 68 chunk->start_addr = virt;
69 chunk->end_addr = virt + size;
66 70
67 write_lock(&pool->lock); 71 write_lock(&pool->lock);
68 list_add(&chunk->next_chunk, &pool->chunks); 72 list_add(&chunk->next_chunk, &pool->chunks);
@@ -70,7 +74,32 @@ int gen_pool_add(struct gen_pool *pool, unsigned long addr, size_t size,
70 74
71 return 0; 75 return 0;
72} 76}
73EXPORT_SYMBOL(gen_pool_add); 77EXPORT_SYMBOL(gen_pool_add_virt);
78
79/**
80 * gen_pool_virt_to_phys - return the physical address of memory
81 * @pool: pool to allocate from
82 * @addr: starting address of memory
83 *
84 * Returns the physical address on success, or -1 on error.
85 */
86phys_addr_t gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long addr)
87{
88 struct list_head *_chunk;
89 struct gen_pool_chunk *chunk;
90
91 read_lock(&pool->lock);
92 list_for_each(_chunk, &pool->chunks) {
93 chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
94
95 if (addr >= chunk->start_addr && addr < chunk->end_addr)
96 return chunk->phys_addr + addr - chunk->start_addr;
97 }
98 read_unlock(&pool->lock);
99
100 return -1;
101}
102EXPORT_SYMBOL(gen_pool_virt_to_phys);
74 103
75/** 104/**
76 * gen_pool_destroy - destroy a special memory pool 105 * gen_pool_destroy - destroy a special memory pool
diff --git a/lib/hexdump.c b/lib/hexdump.c
index 5d7a4802c562..f5fe6ba7a3ab 100644
--- a/lib/hexdump.c
+++ b/lib/hexdump.c
@@ -34,6 +34,22 @@ int hex_to_bin(char ch)
34EXPORT_SYMBOL(hex_to_bin); 34EXPORT_SYMBOL(hex_to_bin);
35 35
36/** 36/**
37 * hex2bin - convert an ascii hexadecimal string to its binary representation
38 * @dst: binary result
39 * @src: ascii hexadecimal string
40 * @count: result length
41 */
42void hex2bin(u8 *dst, const char *src, size_t count)
43{
44 while (count--) {
45 *dst = hex_to_bin(*src++) << 4;
46 *dst += hex_to_bin(*src++);
47 dst++;
48 }
49}
50EXPORT_SYMBOL(hex2bin);
51
52/**
37 * hex_dump_to_buffer - convert a blob of data to "hex ASCII" in memory 53 * hex_dump_to_buffer - convert a blob of data to "hex ASCII" in memory
38 * @buf: data blob to dump 54 * @buf: data blob to dump
39 * @len: number of bytes in the @buf 55 * @len: number of bytes in the @buf
@@ -138,6 +154,7 @@ nil:
138} 154}
139EXPORT_SYMBOL(hex_dump_to_buffer); 155EXPORT_SYMBOL(hex_dump_to_buffer);
140 156
157#ifdef CONFIG_PRINTK
141/** 158/**
142 * print_hex_dump - print a text hex dump to syslog for a binary blob of data 159 * print_hex_dump - print a text hex dump to syslog for a binary blob of data
143 * @level: kernel log level (e.g. KERN_DEBUG) 160 * @level: kernel log level (e.g. KERN_DEBUG)
@@ -222,3 +239,4 @@ void print_hex_dump_bytes(const char *prefix_str, int prefix_type,
222 buf, len, true); 239 buf, len, true);
223} 240}
224EXPORT_SYMBOL(print_hex_dump_bytes); 241EXPORT_SYMBOL(print_hex_dump_bytes);
242#endif
diff --git a/lib/idr.c b/lib/idr.c
index 7f1a4f0acf50..e15502e8b21e 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -106,16 +106,17 @@ static void idr_mark_full(struct idr_layer **pa, int id)
106} 106}
107 107
108/** 108/**
109 * idr_pre_get - reserver resources for idr allocation 109 * idr_pre_get - reserve resources for idr allocation
110 * @idp: idr handle 110 * @idp: idr handle
111 * @gfp_mask: memory allocation flags 111 * @gfp_mask: memory allocation flags
112 * 112 *
113 * This function should be called prior to locking and calling the 113 * This function should be called prior to calling the idr_get_new* functions.
114 * idr_get_new* functions. It preallocates enough memory to satisfy 114 * It preallocates enough memory to satisfy the worst possible allocation. The
115 * the worst possible allocation. 115 * caller should pass in GFP_KERNEL if possible. This of course requires that
116 * no spinning locks be held.
116 * 117 *
117 * If the system is REALLY out of memory this function returns 0, 118 * If the system is REALLY out of memory this function returns %0,
118 * otherwise 1. 119 * otherwise %1.
119 */ 120 */
120int idr_pre_get(struct idr *idp, gfp_t gfp_mask) 121int idr_pre_get(struct idr *idp, gfp_t gfp_mask)
121{ 122{
@@ -284,17 +285,19 @@ static int idr_get_new_above_int(struct idr *idp, void *ptr, int starting_id)
284 * idr_get_new_above - allocate new idr entry above or equal to a start id 285 * idr_get_new_above - allocate new idr entry above or equal to a start id
285 * @idp: idr handle 286 * @idp: idr handle
286 * @ptr: pointer you want associated with the id 287 * @ptr: pointer you want associated with the id
287 * @start_id: id to start search at 288 * @starting_id: id to start search at
288 * @id: pointer to the allocated handle 289 * @id: pointer to the allocated handle
289 * 290 *
290 * This is the allocate id function. It should be called with any 291 * This is the allocate id function. It should be called with any
291 * required locks. 292 * required locks.
292 * 293 *
293 * If memory is required, it will return -EAGAIN, you should unlock 294 * If allocation from IDR's private freelist fails, idr_get_new_above() will
294 * and go back to the idr_pre_get() call. If the idr is full, it will 295 * return %-EAGAIN. The caller should retry the idr_pre_get() call to refill
295 * return -ENOSPC. 296 * IDR's preallocation and then retry the idr_get_new_above() call.
297 *
298 * If the idr is full idr_get_new_above() will return %-ENOSPC.
296 * 299 *
297 * @id returns a value in the range @starting_id ... 0x7fffffff 300 * @id returns a value in the range @starting_id ... %0x7fffffff
298 */ 301 */
299int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id) 302int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id)
300{ 303{
@@ -318,14 +321,13 @@ EXPORT_SYMBOL(idr_get_new_above);
318 * @ptr: pointer you want associated with the id 321 * @ptr: pointer you want associated with the id
319 * @id: pointer to the allocated handle 322 * @id: pointer to the allocated handle
320 * 323 *
321 * This is the allocate id function. It should be called with any 324 * If allocation from IDR's private freelist fails, idr_get_new_above() will
322 * required locks. 325 * return %-EAGAIN. The caller should retry the idr_pre_get() call to refill
326 * IDR's preallocation and then retry the idr_get_new_above() call.
323 * 327 *
324 * If memory is required, it will return -EAGAIN, you should unlock 328 * If the idr is full idr_get_new_above() will return %-ENOSPC.
325 * and go back to the idr_pre_get() call. If the idr is full, it will
326 * return -ENOSPC.
327 * 329 *
328 * @id returns a value in the range 0 ... 0x7fffffff 330 * @id returns a value in the range %0 ... %0x7fffffff
329 */ 331 */
330int idr_get_new(struct idr *idp, void *ptr, int *id) 332int idr_get_new(struct idr *idp, void *ptr, int *id)
331{ 333{
@@ -388,7 +390,7 @@ static void sub_remove(struct idr *idp, int shift, int id)
388} 390}
389 391
390/** 392/**
391 * idr_remove - remove the given id and free it's slot 393 * idr_remove - remove the given id and free its slot
392 * @idp: idr handle 394 * @idp: idr handle
393 * @id: unique key 395 * @id: unique key
394 */ 396 */
@@ -437,7 +439,7 @@ EXPORT_SYMBOL(idr_remove);
437 * function will remove all id mappings and leave all idp_layers 439 * function will remove all id mappings and leave all idp_layers
438 * unused. 440 * unused.
439 * 441 *
440 * A typical clean-up sequence for objects stored in an idr tree, will 442 * A typical clean-up sequence for objects stored in an idr tree will
441 * use idr_for_each() to free all objects, if necessay, then 443 * use idr_for_each() to free all objects, if necessay, then
442 * idr_remove_all() to remove all ids, and idr_destroy() to free 444 * idr_remove_all() to remove all ids, and idr_destroy() to free
443 * up the cached idr_layers. 445 * up the cached idr_layers.
@@ -479,7 +481,7 @@ EXPORT_SYMBOL(idr_remove_all);
479 481
480/** 482/**
481 * idr_destroy - release all cached layers within an idr tree 483 * idr_destroy - release all cached layers within an idr tree
482 * idp: idr handle 484 * @idp: idr handle
483 */ 485 */
484void idr_destroy(struct idr *idp) 486void idr_destroy(struct idr *idp)
485{ 487{
@@ -542,7 +544,7 @@ EXPORT_SYMBOL(idr_find);
542 * not allowed. 544 * not allowed.
543 * 545 *
544 * We check the return of @fn each time. If it returns anything other 546 * We check the return of @fn each time. If it returns anything other
545 * than 0, we break out and return that value. 547 * than %0, we break out and return that value.
546 * 548 *
547 * The caller must serialize idr_for_each() vs idr_get_new() and idr_remove(). 549 * The caller must serialize idr_for_each() vs idr_get_new() and idr_remove().
548 */ 550 */
@@ -586,10 +588,11 @@ EXPORT_SYMBOL(idr_for_each);
586/** 588/**
587 * idr_get_next - lookup next object of id to given id. 589 * idr_get_next - lookup next object of id to given id.
588 * @idp: idr handle 590 * @idp: idr handle
589 * @id: pointer to lookup key 591 * @nextidp: pointer to lookup key
590 * 592 *
591 * Returns pointer to registered object with id, which is next number to 593 * Returns pointer to registered object with id, which is next number to
592 * given id. 594 * given id. After being looked up, *@nextidp will be updated for the next
595 * iteration.
593 */ 596 */
594 597
595void *idr_get_next(struct idr *idp, int *nextidp) 598void *idr_get_next(struct idr *idp, int *nextidp)
@@ -636,8 +639,8 @@ EXPORT_SYMBOL(idr_get_next);
636 * @id: lookup key 639 * @id: lookup key
637 * 640 *
638 * Replace the pointer registered with an id and return the old value. 641 * Replace the pointer registered with an id and return the old value.
639 * A -ENOENT return indicates that @id was not found. 642 * A %-ENOENT return indicates that @id was not found.
640 * A -EINVAL return indicates that @id was not within valid constraints. 643 * A %-EINVAL return indicates that @id was not within valid constraints.
641 * 644 *
642 * The caller must serialize with writers. 645 * The caller must serialize with writers.
643 */ 646 */
@@ -695,10 +698,11 @@ void idr_init(struct idr *idp)
695EXPORT_SYMBOL(idr_init); 698EXPORT_SYMBOL(idr_init);
696 699
697 700
698/* 701/**
702 * DOC: IDA description
699 * IDA - IDR based ID allocator 703 * IDA - IDR based ID allocator
700 * 704 *
701 * this is id allocator without id -> pointer translation. Memory 705 * This is id allocator without id -> pointer translation. Memory
702 * usage is much lower than full blown idr because each id only 706 * usage is much lower than full blown idr because each id only
703 * occupies a bit. ida uses a custom leaf node which contains 707 * occupies a bit. ida uses a custom leaf node which contains
704 * IDA_BITMAP_BITS slots. 708 * IDA_BITMAP_BITS slots.
@@ -731,8 +735,8 @@ static void free_bitmap(struct ida *ida, struct ida_bitmap *bitmap)
731 * following function. It preallocates enough memory to satisfy the 735 * following function. It preallocates enough memory to satisfy the
732 * worst possible allocation. 736 * worst possible allocation.
733 * 737 *
734 * If the system is REALLY out of memory this function returns 0, 738 * If the system is REALLY out of memory this function returns %0,
735 * otherwise 1. 739 * otherwise %1.
736 */ 740 */
737int ida_pre_get(struct ida *ida, gfp_t gfp_mask) 741int ida_pre_get(struct ida *ida, gfp_t gfp_mask)
738{ 742{
@@ -758,17 +762,17 @@ EXPORT_SYMBOL(ida_pre_get);
758/** 762/**
759 * ida_get_new_above - allocate new ID above or equal to a start id 763 * ida_get_new_above - allocate new ID above or equal to a start id
760 * @ida: ida handle 764 * @ida: ida handle
761 * @staring_id: id to start search at 765 * @starting_id: id to start search at
762 * @p_id: pointer to the allocated handle 766 * @p_id: pointer to the allocated handle
763 * 767 *
764 * Allocate new ID above or equal to @ida. It should be called with 768 * Allocate new ID above or equal to @ida. It should be called with
765 * any required locks. 769 * any required locks.
766 * 770 *
767 * If memory is required, it will return -EAGAIN, you should unlock 771 * If memory is required, it will return %-EAGAIN, you should unlock
768 * and go back to the ida_pre_get() call. If the ida is full, it will 772 * and go back to the ida_pre_get() call. If the ida is full, it will
769 * return -ENOSPC. 773 * return %-ENOSPC.
770 * 774 *
771 * @p_id returns a value in the range @starting_id ... 0x7fffffff. 775 * @p_id returns a value in the range @starting_id ... %0x7fffffff.
772 */ 776 */
773int ida_get_new_above(struct ida *ida, int starting_id, int *p_id) 777int ida_get_new_above(struct ida *ida, int starting_id, int *p_id)
774{ 778{
@@ -850,11 +854,11 @@ EXPORT_SYMBOL(ida_get_new_above);
850 * 854 *
851 * Allocate new ID. It should be called with any required locks. 855 * Allocate new ID. It should be called with any required locks.
852 * 856 *
853 * If memory is required, it will return -EAGAIN, you should unlock 857 * If memory is required, it will return %-EAGAIN, you should unlock
854 * and go back to the idr_pre_get() call. If the idr is full, it will 858 * and go back to the idr_pre_get() call. If the idr is full, it will
855 * return -ENOSPC. 859 * return %-ENOSPC.
856 * 860 *
857 * @id returns a value in the range 0 ... 0x7fffffff. 861 * @id returns a value in the range %0 ... %0x7fffffff.
858 */ 862 */
859int ida_get_new(struct ida *ida, int *p_id) 863int ida_get_new(struct ida *ida, int *p_id)
860{ 864{
@@ -912,7 +916,7 @@ EXPORT_SYMBOL(ida_remove);
912 916
913/** 917/**
914 * ida_destroy - release all cached layers within an ida tree 918 * ida_destroy - release all cached layers within an ida tree
915 * ida: ida handle 919 * @ida: ida handle
916 */ 920 */
917void ida_destroy(struct ida *ida) 921void ida_destroy(struct ida *ida)
918{ 922{
diff --git a/lib/ioremap.c b/lib/ioremap.c
index 5730ecd3eb66..da4e2ad74b68 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -9,6 +9,7 @@
9#include <linux/mm.h> 9#include <linux/mm.h>
10#include <linux/sched.h> 10#include <linux/sched.h>
11#include <linux/io.h> 11#include <linux/io.h>
12#include <linux/module.h>
12#include <asm/cacheflush.h> 13#include <asm/cacheflush.h>
13#include <asm/pgtable.h> 14#include <asm/pgtable.h>
14 15
@@ -90,3 +91,4 @@ int ioremap_page_range(unsigned long addr,
90 91
91 return err; 92 return err;
92} 93}
94EXPORT_SYMBOL_GPL(ioremap_page_range);
diff --git a/lib/kernel_lock.c b/lib/kernel_lock.c
deleted file mode 100644
index b135d04aa48a..000000000000
--- a/lib/kernel_lock.c
+++ /dev/null
@@ -1,143 +0,0 @@
1/*
2 * lib/kernel_lock.c
3 *
4 * This is the traditional BKL - big kernel lock. Largely
5 * relegated to obsolescence, but used by various less
6 * important (or lazy) subsystems.
7 */
8#include <linux/module.h>
9#include <linux/kallsyms.h>
10#include <linux/semaphore.h>
11#include <linux/smp_lock.h>
12
13#define CREATE_TRACE_POINTS
14#include <trace/events/bkl.h>
15
16/*
17 * The 'big kernel lock'
18 *
19 * This spinlock is taken and released recursively by lock_kernel()
20 * and unlock_kernel(). It is transparently dropped and reacquired
21 * over schedule(). It is used to protect legacy code that hasn't
22 * been migrated to a proper locking design yet.
23 *
24 * Don't use in new code.
25 */
26static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(kernel_flag);
27
28
29/*
30 * Acquire/release the underlying lock from the scheduler.
31 *
32 * This is called with preemption disabled, and should
33 * return an error value if it cannot get the lock and
34 * TIF_NEED_RESCHED gets set.
35 *
36 * If it successfully gets the lock, it should increment
37 * the preemption count like any spinlock does.
38 *
39 * (This works on UP too - do_raw_spin_trylock will never
40 * return false in that case)
41 */
42int __lockfunc __reacquire_kernel_lock(void)
43{
44 while (!do_raw_spin_trylock(&kernel_flag)) {
45 if (need_resched())
46 return -EAGAIN;
47 cpu_relax();
48 }
49 preempt_disable();
50 return 0;
51}
52
53void __lockfunc __release_kernel_lock(void)
54{
55 do_raw_spin_unlock(&kernel_flag);
56 preempt_enable_no_resched();
57}
58
59/*
60 * These are the BKL spinlocks - we try to be polite about preemption.
61 * If SMP is not on (ie UP preemption), this all goes away because the
62 * do_raw_spin_trylock() will always succeed.
63 */
64#ifdef CONFIG_PREEMPT
65static inline void __lock_kernel(void)
66{
67 preempt_disable();
68 if (unlikely(!do_raw_spin_trylock(&kernel_flag))) {
69 /*
70 * If preemption was disabled even before this
71 * was called, there's nothing we can be polite
72 * about - just spin.
73 */
74 if (preempt_count() > 1) {
75 do_raw_spin_lock(&kernel_flag);
76 return;
77 }
78
79 /*
80 * Otherwise, let's wait for the kernel lock
81 * with preemption enabled..
82 */
83 do {
84 preempt_enable();
85 while (raw_spin_is_locked(&kernel_flag))
86 cpu_relax();
87 preempt_disable();
88 } while (!do_raw_spin_trylock(&kernel_flag));
89 }
90}
91
92#else
93
94/*
95 * Non-preemption case - just get the spinlock
96 */
97static inline void __lock_kernel(void)
98{
99 do_raw_spin_lock(&kernel_flag);
100}
101#endif
102
103static inline void __unlock_kernel(void)
104{
105 /*
106 * the BKL is not covered by lockdep, so we open-code the
107 * unlocking sequence (and thus avoid the dep-chain ops):
108 */
109 do_raw_spin_unlock(&kernel_flag);
110 preempt_enable();
111}
112
113/*
114 * Getting the big kernel lock.
115 *
116 * This cannot happen asynchronously, so we only need to
117 * worry about other CPU's.
118 */
119void __lockfunc _lock_kernel(const char *func, const char *file, int line)
120{
121 int depth = current->lock_depth + 1;
122
123 trace_lock_kernel(func, file, line);
124
125 if (likely(!depth)) {
126 might_sleep();
127 __lock_kernel();
128 }
129 current->lock_depth = depth;
130}
131
132void __lockfunc _unlock_kernel(const char *func, const char *file, int line)
133{
134 BUG_ON(current->lock_depth < 0);
135 if (likely(--current->lock_depth < 0))
136 __unlock_kernel();
137
138 trace_unlock_kernel(func, file, line);
139}
140
141EXPORT_SYMBOL(_lock_kernel);
142EXPORT_SYMBOL(_unlock_kernel);
143
diff --git a/lib/kobject.c b/lib/kobject.c
index f07c57252e82..640bd98a4c8a 100644
--- a/lib/kobject.c
+++ b/lib/kobject.c
@@ -746,17 +746,56 @@ void kset_unregister(struct kset *k)
746 */ 746 */
747struct kobject *kset_find_obj(struct kset *kset, const char *name) 747struct kobject *kset_find_obj(struct kset *kset, const char *name)
748{ 748{
749 return kset_find_obj_hinted(kset, name, NULL);
750}
751
752/**
753 * kset_find_obj_hinted - search for object in kset given a predecessor hint.
754 * @kset: kset we're looking in.
755 * @name: object's name.
756 * @hint: hint to possible object's predecessor.
757 *
758 * Check the hint's next object and if it is a match return it directly,
759 * otherwise, fall back to the behavior of kset_find_obj(). Either way
760 * a reference for the returned object is held and the reference on the
761 * hinted object is released.
762 */
763struct kobject *kset_find_obj_hinted(struct kset *kset, const char *name,
764 struct kobject *hint)
765{
749 struct kobject *k; 766 struct kobject *k;
750 struct kobject *ret = NULL; 767 struct kobject *ret = NULL;
751 768
752 spin_lock(&kset->list_lock); 769 spin_lock(&kset->list_lock);
770
771 if (!hint)
772 goto slow_search;
773
774 /* end of list detection */
775 if (hint->entry.next == kset->list.next)
776 goto slow_search;
777
778 k = container_of(hint->entry.next, struct kobject, entry);
779 if (!kobject_name(k) || strcmp(kobject_name(k), name))
780 goto slow_search;
781
782 ret = kobject_get(k);
783 goto unlock_exit;
784
785slow_search:
753 list_for_each_entry(k, &kset->list, entry) { 786 list_for_each_entry(k, &kset->list, entry) {
754 if (kobject_name(k) && !strcmp(kobject_name(k), name)) { 787 if (kobject_name(k) && !strcmp(kobject_name(k), name)) {
755 ret = kobject_get(k); 788 ret = kobject_get(k);
756 break; 789 break;
757 } 790 }
758 } 791 }
792
793unlock_exit:
759 spin_unlock(&kset->list_lock); 794 spin_unlock(&kset->list_lock);
795
796 if (hint)
797 kobject_put(hint);
798
760 return ret; 799 return ret;
761} 800}
762 801
@@ -909,14 +948,14 @@ const struct kobj_ns_type_operations *kobj_ns_ops(struct kobject *kobj)
909} 948}
910 949
911 950
912const void *kobj_ns_current(enum kobj_ns_type type) 951void *kobj_ns_grab_current(enum kobj_ns_type type)
913{ 952{
914 const void *ns = NULL; 953 void *ns = NULL;
915 954
916 spin_lock(&kobj_ns_type_lock); 955 spin_lock(&kobj_ns_type_lock);
917 if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) && 956 if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) &&
918 kobj_ns_ops_tbl[type]) 957 kobj_ns_ops_tbl[type])
919 ns = kobj_ns_ops_tbl[type]->current_ns(); 958 ns = kobj_ns_ops_tbl[type]->grab_current_ns();
920 spin_unlock(&kobj_ns_type_lock); 959 spin_unlock(&kobj_ns_type_lock);
921 960
922 return ns; 961 return ns;
@@ -948,23 +987,15 @@ const void *kobj_ns_initial(enum kobj_ns_type type)
948 return ns; 987 return ns;
949} 988}
950 989
951/* 990void kobj_ns_drop(enum kobj_ns_type type, void *ns)
952 * kobj_ns_exit - invalidate a namespace tag
953 *
954 * @type: the namespace type (i.e. KOBJ_NS_TYPE_NET)
955 * @ns: the actual namespace being invalidated
956 *
957 * This is called when a tag is no longer valid. For instance,
958 * when a network namespace exits, it uses this helper to
959 * make sure no sb's sysfs_info points to the now-invalidated
960 * netns.
961 */
962void kobj_ns_exit(enum kobj_ns_type type, const void *ns)
963{ 991{
964 sysfs_exit_ns(type, ns); 992 spin_lock(&kobj_ns_type_lock);
993 if ((type > KOBJ_NS_TYPE_NONE) && (type < KOBJ_NS_TYPES) &&
994 kobj_ns_ops_tbl[type] && kobj_ns_ops_tbl[type]->drop_ns)
995 kobj_ns_ops_tbl[type]->drop_ns(ns);
996 spin_unlock(&kobj_ns_type_lock);
965} 997}
966 998
967
968EXPORT_SYMBOL(kobject_get); 999EXPORT_SYMBOL(kobject_get);
969EXPORT_SYMBOL(kobject_put); 1000EXPORT_SYMBOL(kobject_put);
970EXPORT_SYMBOL(kobject_del); 1001EXPORT_SYMBOL(kobject_del);
diff --git a/lib/kref.c b/lib/kref.c
index d3d227a08a4b..3efb882b11db 100644
--- a/lib/kref.c
+++ b/lib/kref.c
@@ -62,6 +62,36 @@ int kref_put(struct kref *kref, void (*release)(struct kref *kref))
62 return 0; 62 return 0;
63} 63}
64 64
65
66/**
67 * kref_sub - subtract a number of refcounts for object.
68 * @kref: object.
69 * @count: Number of recounts to subtract.
70 * @release: pointer to the function that will clean up the object when the
71 * last reference to the object is released.
72 * This pointer is required, and it is not acceptable to pass kfree
73 * in as this function.
74 *
75 * Subtract @count from the refcount, and if 0, call release().
76 * Return 1 if the object was removed, otherwise return 0. Beware, if this
77 * function returns 0, you still can not count on the kref from remaining in
78 * memory. Only use the return value if you want to see if the kref is now
79 * gone, not present.
80 */
81int kref_sub(struct kref *kref, unsigned int count,
82 void (*release)(struct kref *kref))
83{
84 WARN_ON(release == NULL);
85 WARN_ON(release == (void (*)(struct kref *))kfree);
86
87 if (atomic_sub_and_test((int) count, &kref->refcount)) {
88 release(kref);
89 return 1;
90 }
91 return 0;
92}
93
65EXPORT_SYMBOL(kref_init); 94EXPORT_SYMBOL(kref_init);
66EXPORT_SYMBOL(kref_get); 95EXPORT_SYMBOL(kref_get);
67EXPORT_SYMBOL(kref_put); 96EXPORT_SYMBOL(kref_put);
97EXPORT_SYMBOL(kref_sub);
diff --git a/lib/kstrtox.c b/lib/kstrtox.c
new file mode 100644
index 000000000000..2dbae88090ac
--- /dev/null
+++ b/lib/kstrtox.c
@@ -0,0 +1,250 @@
1/*
2 * Convert integer string representation to an integer.
3 * If an integer doesn't fit into specified type, -E is returned.
4 *
5 * Integer starts with optional sign.
6 * kstrtou*() functions do not accept sign "-".
7 *
8 * Radix 0 means autodetection: leading "0x" implies radix 16,
9 * leading "0" implies radix 8, otherwise radix is 10.
10 * Autodetection hints work after optional sign, but not before.
11 *
12 * If -E is returned, result is not touched.
13 */
14#include <linux/ctype.h>
15#include <linux/errno.h>
16#include <linux/kernel.h>
17#include <linux/math64.h>
18#include <linux/module.h>
19#include <linux/types.h>
20#include <asm/uaccess.h>
21
22static inline char _tolower(const char c)
23{
24 return c | 0x20;
25}
26
27static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res)
28{
29 unsigned long long acc;
30 int ok;
31
32 if (base == 0) {
33 if (s[0] == '0') {
34 if (_tolower(s[1]) == 'x' && isxdigit(s[2]))
35 base = 16;
36 else
37 base = 8;
38 } else
39 base = 10;
40 }
41 if (base == 16 && s[0] == '0' && _tolower(s[1]) == 'x')
42 s += 2;
43
44 acc = 0;
45 ok = 0;
46 while (*s) {
47 unsigned int val;
48
49 if ('0' <= *s && *s <= '9')
50 val = *s - '0';
51 else if ('a' <= _tolower(*s) && _tolower(*s) <= 'f')
52 val = _tolower(*s) - 'a' + 10;
53 else if (*s == '\n' && *(s + 1) == '\0')
54 break;
55 else
56 return -EINVAL;
57
58 if (val >= base)
59 return -EINVAL;
60 if (acc > div_u64(ULLONG_MAX - val, base))
61 return -ERANGE;
62 acc = acc * base + val;
63 ok = 1;
64
65 s++;
66 }
67 if (!ok)
68 return -EINVAL;
69 *res = acc;
70 return 0;
71}
72
73int kstrtoull(const char *s, unsigned int base, unsigned long long *res)
74{
75 if (s[0] == '+')
76 s++;
77 return _kstrtoull(s, base, res);
78}
79EXPORT_SYMBOL(kstrtoull);
80
81int kstrtoll(const char *s, unsigned int base, long long *res)
82{
83 unsigned long long tmp;
84 int rv;
85
86 if (s[0] == '-') {
87 rv = _kstrtoull(s + 1, base, &tmp);
88 if (rv < 0)
89 return rv;
90 if ((long long)(-tmp) >= 0)
91 return -ERANGE;
92 *res = -tmp;
93 } else {
94 rv = kstrtoull(s, base, &tmp);
95 if (rv < 0)
96 return rv;
97 if ((long long)tmp < 0)
98 return -ERANGE;
99 *res = tmp;
100 }
101 return 0;
102}
103EXPORT_SYMBOL(kstrtoll);
104
105/* Internal, do not use. */
106int _kstrtoul(const char *s, unsigned int base, unsigned long *res)
107{
108 unsigned long long tmp;
109 int rv;
110
111 rv = kstrtoull(s, base, &tmp);
112 if (rv < 0)
113 return rv;
114 if (tmp != (unsigned long long)(unsigned long)tmp)
115 return -ERANGE;
116 *res = tmp;
117 return 0;
118}
119EXPORT_SYMBOL(_kstrtoul);
120
121/* Internal, do not use. */
122int _kstrtol(const char *s, unsigned int base, long *res)
123{
124 long long tmp;
125 int rv;
126
127 rv = kstrtoll(s, base, &tmp);
128 if (rv < 0)
129 return rv;
130 if (tmp != (long long)(long)tmp)
131 return -ERANGE;
132 *res = tmp;
133 return 0;
134}
135EXPORT_SYMBOL(_kstrtol);
136
137int kstrtouint(const char *s, unsigned int base, unsigned int *res)
138{
139 unsigned long long tmp;
140 int rv;
141
142 rv = kstrtoull(s, base, &tmp);
143 if (rv < 0)
144 return rv;
145 if (tmp != (unsigned long long)(unsigned int)tmp)
146 return -ERANGE;
147 *res = tmp;
148 return 0;
149}
150EXPORT_SYMBOL(kstrtouint);
151
152int kstrtoint(const char *s, unsigned int base, int *res)
153{
154 long long tmp;
155 int rv;
156
157 rv = kstrtoll(s, base, &tmp);
158 if (rv < 0)
159 return rv;
160 if (tmp != (long long)(int)tmp)
161 return -ERANGE;
162 *res = tmp;
163 return 0;
164}
165EXPORT_SYMBOL(kstrtoint);
166
167int kstrtou16(const char *s, unsigned int base, u16 *res)
168{
169 unsigned long long tmp;
170 int rv;
171
172 rv = kstrtoull(s, base, &tmp);
173 if (rv < 0)
174 return rv;
175 if (tmp != (unsigned long long)(u16)tmp)
176 return -ERANGE;
177 *res = tmp;
178 return 0;
179}
180EXPORT_SYMBOL(kstrtou16);
181
182int kstrtos16(const char *s, unsigned int base, s16 *res)
183{
184 long long tmp;
185 int rv;
186
187 rv = kstrtoll(s, base, &tmp);
188 if (rv < 0)
189 return rv;
190 if (tmp != (long long)(s16)tmp)
191 return -ERANGE;
192 *res = tmp;
193 return 0;
194}
195EXPORT_SYMBOL(kstrtos16);
196
197int kstrtou8(const char *s, unsigned int base, u8 *res)
198{
199 unsigned long long tmp;
200 int rv;
201
202 rv = kstrtoull(s, base, &tmp);
203 if (rv < 0)
204 return rv;
205 if (tmp != (unsigned long long)(u8)tmp)
206 return -ERANGE;
207 *res = tmp;
208 return 0;
209}
210EXPORT_SYMBOL(kstrtou8);
211
212int kstrtos8(const char *s, unsigned int base, s8 *res)
213{
214 long long tmp;
215 int rv;
216
217 rv = kstrtoll(s, base, &tmp);
218 if (rv < 0)
219 return rv;
220 if (tmp != (long long)(s8)tmp)
221 return -ERANGE;
222 *res = tmp;
223 return 0;
224}
225EXPORT_SYMBOL(kstrtos8);
226
227#define kstrto_from_user(f, g, type) \
228int f(const char __user *s, size_t count, unsigned int base, type *res) \
229{ \
230 /* sign, base 2 representation, newline, terminator */ \
231 char buf[1 + sizeof(type) * 8 + 1 + 1]; \
232 \
233 count = min(count, sizeof(buf) - 1); \
234 if (copy_from_user(buf, s, count)) \
235 return -EFAULT; \
236 buf[count] = '\0'; \
237 return g(buf, base, res); \
238} \
239EXPORT_SYMBOL(f)
240
241kstrto_from_user(kstrtoull_from_user, kstrtoull, unsigned long long);
242kstrto_from_user(kstrtoll_from_user, kstrtoll, long long);
243kstrto_from_user(kstrtoul_from_user, kstrtoul, unsigned long);
244kstrto_from_user(kstrtol_from_user, kstrtol, long);
245kstrto_from_user(kstrtouint_from_user, kstrtouint, unsigned int);
246kstrto_from_user(kstrtoint_from_user, kstrtoint, int);
247kstrto_from_user(kstrtou16_from_user, kstrtou16, u16);
248kstrto_from_user(kstrtos16_from_user, kstrtos16, s16);
249kstrto_from_user(kstrtou8_from_user, kstrtou8, u8);
250kstrto_from_user(kstrtos8_from_user, kstrtos8, s8);
diff --git a/lib/list_debug.c b/lib/list_debug.c
index 344c710d16ca..b8029a5583ff 100644
--- a/lib/list_debug.c
+++ b/lib/list_debug.c
@@ -35,6 +35,31 @@ void __list_add(struct list_head *new,
35} 35}
36EXPORT_SYMBOL(__list_add); 36EXPORT_SYMBOL(__list_add);
37 37
38void __list_del_entry(struct list_head *entry)
39{
40 struct list_head *prev, *next;
41
42 prev = entry->prev;
43 next = entry->next;
44
45 if (WARN(next == LIST_POISON1,
46 "list_del corruption, %p->next is LIST_POISON1 (%p)\n",
47 entry, LIST_POISON1) ||
48 WARN(prev == LIST_POISON2,
49 "list_del corruption, %p->prev is LIST_POISON2 (%p)\n",
50 entry, LIST_POISON2) ||
51 WARN(prev->next != entry,
52 "list_del corruption. prev->next should be %p, "
53 "but was %p\n", entry, prev->next) ||
54 WARN(next->prev != entry,
55 "list_del corruption. next->prev should be %p, "
56 "but was %p\n", entry, next->prev))
57 return;
58
59 __list_del(prev, next);
60}
61EXPORT_SYMBOL(__list_del_entry);
62
38/** 63/**
39 * list_del - deletes entry from list. 64 * list_del - deletes entry from list.
40 * @entry: the element to delete from the list. 65 * @entry: the element to delete from the list.
@@ -43,19 +68,7 @@ EXPORT_SYMBOL(__list_add);
43 */ 68 */
44void list_del(struct list_head *entry) 69void list_del(struct list_head *entry)
45{ 70{
46 WARN(entry->next == LIST_POISON1, 71 __list_del_entry(entry);
47 "list_del corruption, next is LIST_POISON1 (%p)\n",
48 LIST_POISON1);
49 WARN(entry->next != LIST_POISON1 && entry->prev == LIST_POISON2,
50 "list_del corruption, prev is LIST_POISON2 (%p)\n",
51 LIST_POISON2);
52 WARN(entry->prev->next != entry,
53 "list_del corruption. prev->next should be %p, "
54 "but was %p\n", entry, entry->prev->next);
55 WARN(entry->next->prev != entry,
56 "list_del corruption. next->prev should be %p, "
57 "but was %p\n", entry, entry->next->prev);
58 __list_del(entry->prev, entry->next);
59 entry->next = LIST_POISON1; 72 entry->next = LIST_POISON1;
60 entry->prev = LIST_POISON2; 73 entry->prev = LIST_POISON2;
61} 74}
diff --git a/lib/list_sort.c b/lib/list_sort.c
index a7616fa3162e..d7325c6b103f 100644
--- a/lib/list_sort.c
+++ b/lib/list_sort.c
@@ -141,77 +141,151 @@ void list_sort(void *priv, struct list_head *head,
141} 141}
142EXPORT_SYMBOL(list_sort); 142EXPORT_SYMBOL(list_sort);
143 143
144#ifdef DEBUG_LIST_SORT 144#ifdef CONFIG_TEST_LIST_SORT
145
146#include <linux/random.h>
147
148/*
149 * The pattern of set bits in the list length determines which cases
150 * are hit in list_sort().
151 */
152#define TEST_LIST_LEN (512+128+2) /* not including head */
153
154#define TEST_POISON1 0xDEADBEEF
155#define TEST_POISON2 0xA324354C
156
145struct debug_el { 157struct debug_el {
146 struct list_head l_h; 158 unsigned int poison1;
159 struct list_head list;
160 unsigned int poison2;
147 int value; 161 int value;
148 unsigned serial; 162 unsigned serial;
149}; 163};
150 164
151static int cmp(void *priv, struct list_head *a, struct list_head *b) 165/* Array, containing pointers to all elements in the test list */
166static struct debug_el **elts __initdata;
167
168static int __init check(struct debug_el *ela, struct debug_el *elb)
152{ 169{
153 return container_of(a, struct debug_el, l_h)->value 170 if (ela->serial >= TEST_LIST_LEN) {
154 - container_of(b, struct debug_el, l_h)->value; 171 printk(KERN_ERR "list_sort_test: error: incorrect serial %d\n",
172 ela->serial);
173 return -EINVAL;
174 }
175 if (elb->serial >= TEST_LIST_LEN) {
176 printk(KERN_ERR "list_sort_test: error: incorrect serial %d\n",
177 elb->serial);
178 return -EINVAL;
179 }
180 if (elts[ela->serial] != ela || elts[elb->serial] != elb) {
181 printk(KERN_ERR "list_sort_test: error: phantom element\n");
182 return -EINVAL;
183 }
184 if (ela->poison1 != TEST_POISON1 || ela->poison2 != TEST_POISON2) {
185 printk(KERN_ERR "list_sort_test: error: bad poison: %#x/%#x\n",
186 ela->poison1, ela->poison2);
187 return -EINVAL;
188 }
189 if (elb->poison1 != TEST_POISON1 || elb->poison2 != TEST_POISON2) {
190 printk(KERN_ERR "list_sort_test: error: bad poison: %#x/%#x\n",
191 elb->poison1, elb->poison2);
192 return -EINVAL;
193 }
194 return 0;
155} 195}
156 196
157/* 197static int __init cmp(void *priv, struct list_head *a, struct list_head *b)
158 * The pattern of set bits in the list length determines which cases 198{
159 * are hit in list_sort(). 199 struct debug_el *ela, *elb;
160 */ 200
161#define LIST_SORT_TEST_LENGTH (512+128+2) /* not including head */ 201 ela = container_of(a, struct debug_el, list);
202 elb = container_of(b, struct debug_el, list);
203
204 check(ela, elb);
205 return ela->value - elb->value;
206}
162 207
163static int __init list_sort_test(void) 208static int __init list_sort_test(void)
164{ 209{
165 int i, r = 1, count; 210 int i, count = 1, err = -EINVAL;
166 struct list_head *head = kmalloc(sizeof(*head), GFP_KERNEL); 211 struct debug_el *el;
167 struct list_head *cur; 212 struct list_head *cur, *tmp;
213 LIST_HEAD(head);
214
215 printk(KERN_DEBUG "list_sort_test: start testing list_sort()\n");
168 216
169 printk(KERN_WARNING "testing list_sort()\n"); 217 elts = kmalloc(sizeof(void *) * TEST_LIST_LEN, GFP_KERNEL);
218 if (!elts) {
219 printk(KERN_ERR "list_sort_test: error: cannot allocate "
220 "memory\n");
221 goto exit;
222 }
170 223
171 cur = head; 224 for (i = 0; i < TEST_LIST_LEN; i++) {
172 for (i = 0; i < LIST_SORT_TEST_LENGTH; i++) { 225 el = kmalloc(sizeof(*el), GFP_KERNEL);
173 struct debug_el *el = kmalloc(sizeof(*el), GFP_KERNEL); 226 if (!el) {
174 BUG_ON(!el); 227 printk(KERN_ERR "list_sort_test: error: cannot "
228 "allocate memory\n");
229 goto exit;
230 }
175 /* force some equivalencies */ 231 /* force some equivalencies */
176 el->value = (r = (r * 725861) % 6599) % (LIST_SORT_TEST_LENGTH/3); 232 el->value = random32() % (TEST_LIST_LEN/3);
177 el->serial = i; 233 el->serial = i;
178 234 el->poison1 = TEST_POISON1;
179 el->l_h.prev = cur; 235 el->poison2 = TEST_POISON2;
180 cur->next = &el->l_h; 236 elts[i] = el;
181 cur = cur->next; 237 list_add_tail(&el->list, &head);
182 } 238 }
183 head->prev = cur;
184 239
185 list_sort(NULL, head, cmp); 240 list_sort(NULL, &head, cmp);
241
242 for (cur = head.next; cur->next != &head; cur = cur->next) {
243 struct debug_el *el1;
244 int cmp_result;
186 245
187 count = 1;
188 for (cur = head->next; cur->next != head; cur = cur->next) {
189 struct debug_el *el = container_of(cur, struct debug_el, l_h);
190 int cmp_result = cmp(NULL, cur, cur->next);
191 if (cur->next->prev != cur) { 246 if (cur->next->prev != cur) {
192 printk(KERN_EMERG "list_sort() returned " 247 printk(KERN_ERR "list_sort_test: error: list is "
193 "a corrupted list!\n"); 248 "corrupted\n");
194 return 1; 249 goto exit;
195 } else if (cmp_result > 0) { 250 }
196 printk(KERN_EMERG "list_sort() failed to sort!\n"); 251
197 return 1; 252 cmp_result = cmp(NULL, cur, cur->next);
198 } else if (cmp_result == 0 && 253 if (cmp_result > 0) {
199 el->serial >= container_of(cur->next, 254 printk(KERN_ERR "list_sort_test: error: list is not "
200 struct debug_el, l_h)->serial) { 255 "sorted\n");
201 printk(KERN_EMERG "list_sort() failed to preserve order" 256 goto exit;
202 " of equivalent elements!\n"); 257 }
203 return 1; 258
259 el = container_of(cur, struct debug_el, list);
260 el1 = container_of(cur->next, struct debug_el, list);
261 if (cmp_result == 0 && el->serial >= el1->serial) {
262 printk(KERN_ERR "list_sort_test: error: order of "
263 "equivalent elements not preserved\n");
264 goto exit;
265 }
266
267 if (check(el, el1)) {
268 printk(KERN_ERR "list_sort_test: error: element check "
269 "failed\n");
270 goto exit;
204 } 271 }
205 kfree(cur->prev);
206 count++; 272 count++;
207 } 273 }
208 kfree(cur); 274
209 if (count != LIST_SORT_TEST_LENGTH) { 275 if (count != TEST_LIST_LEN) {
210 printk(KERN_EMERG "list_sort() returned list of" 276 printk(KERN_ERR "list_sort_test: error: bad list length %d",
211 "different length!\n"); 277 count);
212 return 1; 278 goto exit;
213 } 279 }
214 return 0; 280
281 err = 0;
282exit:
283 kfree(elts);
284 list_for_each_safe(cur, tmp, &head) {
285 list_del(cur);
286 kfree(container_of(cur, struct debug_el, list));
287 }
288 return err;
215} 289}
216module_init(list_sort_test); 290module_init(list_sort_test);
217#endif 291#endif /* CONFIG_TEST_LIST_SORT */
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index 619313ed6c46..507a22fab738 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -144,7 +144,7 @@ static void init_shared_classes(void)
144 144
145#define HARDIRQ_ENTER() \ 145#define HARDIRQ_ENTER() \
146 local_irq_disable(); \ 146 local_irq_disable(); \
147 irq_enter(); \ 147 __irq_enter(); \
148 WARN_ON(!in_irq()); 148 WARN_ON(!in_irq());
149 149
150#define HARDIRQ_EXIT() \ 150#define HARDIRQ_EXIT() \
diff --git a/lib/lru_cache.c b/lib/lru_cache.c
index 270de9d31b8c..a07e7268d7ed 100644
--- a/lib/lru_cache.c
+++ b/lib/lru_cache.c
@@ -84,7 +84,7 @@ struct lru_cache *lc_create(const char *name, struct kmem_cache *cache,
84 if (e_count > LC_MAX_ACTIVE) 84 if (e_count > LC_MAX_ACTIVE)
85 return NULL; 85 return NULL;
86 86
87 slot = kzalloc(e_count * sizeof(struct hlist_head*), GFP_KERNEL); 87 slot = kcalloc(e_count, sizeof(struct hlist_head), GFP_KERNEL);
88 if (!slot) 88 if (!slot)
89 goto out_fail; 89 goto out_fail;
90 element = kzalloc(e_count * sizeof(struct lc_element *), GFP_KERNEL); 90 element = kzalloc(e_count * sizeof(struct lc_element *), GFP_KERNEL);
diff --git a/lib/nlattr.c b/lib/nlattr.c
index c4706eb98d3d..ac09f2226dc7 100644
--- a/lib/nlattr.c
+++ b/lib/nlattr.c
@@ -15,7 +15,7 @@
15#include <linux/types.h> 15#include <linux/types.h>
16#include <net/netlink.h> 16#include <net/netlink.h>
17 17
18static u16 nla_attr_minlen[NLA_TYPE_MAX+1] __read_mostly = { 18static const u16 nla_attr_minlen[NLA_TYPE_MAX+1] = {
19 [NLA_U8] = sizeof(u8), 19 [NLA_U8] = sizeof(u8),
20 [NLA_U16] = sizeof(u16), 20 [NLA_U16] = sizeof(u16),
21 [NLA_U32] = sizeof(u32), 21 [NLA_U32] = sizeof(u32),
@@ -23,7 +23,7 @@ static u16 nla_attr_minlen[NLA_TYPE_MAX+1] __read_mostly = {
23 [NLA_NESTED] = NLA_HDRLEN, 23 [NLA_NESTED] = NLA_HDRLEN,
24}; 24};
25 25
26static int validate_nla(struct nlattr *nla, int maxtype, 26static int validate_nla(const struct nlattr *nla, int maxtype,
27 const struct nla_policy *policy) 27 const struct nla_policy *policy)
28{ 28{
29 const struct nla_policy *pt; 29 const struct nla_policy *pt;
@@ -115,10 +115,10 @@ static int validate_nla(struct nlattr *nla, int maxtype,
115 * 115 *
116 * Returns 0 on success or a negative error code. 116 * Returns 0 on success or a negative error code.
117 */ 117 */
118int nla_validate(struct nlattr *head, int len, int maxtype, 118int nla_validate(const struct nlattr *head, int len, int maxtype,
119 const struct nla_policy *policy) 119 const struct nla_policy *policy)
120{ 120{
121 struct nlattr *nla; 121 const struct nlattr *nla;
122 int rem, err; 122 int rem, err;
123 123
124 nla_for_each_attr(nla, head, len, rem) { 124 nla_for_each_attr(nla, head, len, rem) {
@@ -148,7 +148,7 @@ nla_policy_len(const struct nla_policy *p, int n)
148{ 148{
149 int i, len = 0; 149 int i, len = 0;
150 150
151 for (i = 0; i < n; i++) { 151 for (i = 0; i < n; i++, p++) {
152 if (p->len) 152 if (p->len)
153 len += nla_total_size(p->len); 153 len += nla_total_size(p->len);
154 else if (nla_attr_minlen[p->type]) 154 else if (nla_attr_minlen[p->type])
@@ -167,16 +167,16 @@ nla_policy_len(const struct nla_policy *p, int n)
167 * @policy: validation policy 167 * @policy: validation policy
168 * 168 *
169 * Parses a stream of attributes and stores a pointer to each attribute in 169 * Parses a stream of attributes and stores a pointer to each attribute in
170 * the tb array accessable via the attribute type. Attributes with a type 170 * the tb array accessible via the attribute type. Attributes with a type
171 * exceeding maxtype will be silently ignored for backwards compatibility 171 * exceeding maxtype will be silently ignored for backwards compatibility
172 * reasons. policy may be set to NULL if no validation is required. 172 * reasons. policy may be set to NULL if no validation is required.
173 * 173 *
174 * Returns 0 on success or a negative error code. 174 * Returns 0 on success or a negative error code.
175 */ 175 */
176int nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, int len, 176int nla_parse(struct nlattr **tb, int maxtype, const struct nlattr *head,
177 const struct nla_policy *policy) 177 int len, const struct nla_policy *policy)
178{ 178{
179 struct nlattr *nla; 179 const struct nlattr *nla;
180 int rem, err; 180 int rem, err;
181 181
182 memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1)); 182 memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1));
@@ -191,7 +191,7 @@ int nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, int len,
191 goto errout; 191 goto errout;
192 } 192 }
193 193
194 tb[type] = nla; 194 tb[type] = (struct nlattr *)nla;
195 } 195 }
196 } 196 }
197 197
@@ -212,14 +212,14 @@ errout:
212 * 212 *
213 * Returns the first attribute in the stream matching the specified type. 213 * Returns the first attribute in the stream matching the specified type.
214 */ 214 */
215struct nlattr *nla_find(struct nlattr *head, int len, int attrtype) 215struct nlattr *nla_find(const struct nlattr *head, int len, int attrtype)
216{ 216{
217 struct nlattr *nla; 217 const struct nlattr *nla;
218 int rem; 218 int rem;
219 219
220 nla_for_each_attr(nla, head, len, rem) 220 nla_for_each_attr(nla, head, len, rem)
221 if (nla_type(nla) == attrtype) 221 if (nla_type(nla) == attrtype)
222 return nla; 222 return (struct nlattr *)nla;
223 223
224 return NULL; 224 return NULL;
225} 225}
diff --git a/lib/parser.c b/lib/parser.c
index fb34977246bb..dcbaaef6cf11 100644
--- a/lib/parser.c
+++ b/lib/parser.c
@@ -13,7 +13,7 @@
13 13
14/** 14/**
15 * match_one: - Determines if a string matches a simple pattern 15 * match_one: - Determines if a string matches a simple pattern
16 * @s: the string to examine for presense of the pattern 16 * @s: the string to examine for presence of the pattern
17 * @p: the string containing the pattern 17 * @p: the string containing the pattern
18 * @args: array of %MAX_OPT_ARGS &substring_t elements. Used to return match 18 * @args: array of %MAX_OPT_ARGS &substring_t elements. Used to return match
19 * locations. 19 * locations.
@@ -128,12 +128,13 @@ static int match_number(substring_t *s, int *result, int base)
128 char *endp; 128 char *endp;
129 char *buf; 129 char *buf;
130 int ret; 130 int ret;
131 size_t len = s->to - s->from;
131 132
132 buf = kmalloc(s->to - s->from + 1, GFP_KERNEL); 133 buf = kmalloc(len + 1, GFP_KERNEL);
133 if (!buf) 134 if (!buf)
134 return -ENOMEM; 135 return -ENOMEM;
135 memcpy(buf, s->from, s->to - s->from); 136 memcpy(buf, s->from, len);
136 buf[s->to - s->from] = '\0'; 137 buf[len] = '\0';
137 *result = simple_strtol(buf, &endp, base); 138 *result = simple_strtol(buf, &endp, base);
138 ret = 0; 139 ret = 0;
139 if (endp == buf) 140 if (endp == buf)
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index ec9048e74f44..28f2c33c6b53 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -8,10 +8,53 @@
8#include <linux/init.h> 8#include <linux/init.h>
9#include <linux/cpu.h> 9#include <linux/cpu.h>
10#include <linux/module.h> 10#include <linux/module.h>
11#include <linux/debugobjects.h>
11 12
12static LIST_HEAD(percpu_counters); 13static LIST_HEAD(percpu_counters);
13static DEFINE_MUTEX(percpu_counters_lock); 14static DEFINE_MUTEX(percpu_counters_lock);
14 15
16#ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
17
18static struct debug_obj_descr percpu_counter_debug_descr;
19
20static int percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
21{
22 struct percpu_counter *fbc = addr;
23
24 switch (state) {
25 case ODEBUG_STATE_ACTIVE:
26 percpu_counter_destroy(fbc);
27 debug_object_free(fbc, &percpu_counter_debug_descr);
28 return 1;
29 default:
30 return 0;
31 }
32}
33
34static struct debug_obj_descr percpu_counter_debug_descr = {
35 .name = "percpu_counter",
36 .fixup_free = percpu_counter_fixup_free,
37};
38
39static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
40{
41 debug_object_init(fbc, &percpu_counter_debug_descr);
42 debug_object_activate(fbc, &percpu_counter_debug_descr);
43}
44
45static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
46{
47 debug_object_deactivate(fbc, &percpu_counter_debug_descr);
48 debug_object_free(fbc, &percpu_counter_debug_descr);
49}
50
51#else /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
52static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
53{ }
54static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
55{ }
56#endif /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
57
15void percpu_counter_set(struct percpu_counter *fbc, s64 amount) 58void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
16{ 59{
17 int cpu; 60 int cpu;
@@ -29,20 +72,18 @@ EXPORT_SYMBOL(percpu_counter_set);
29void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch) 72void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
30{ 73{
31 s64 count; 74 s64 count;
32 s32 *pcount;
33 int cpu = get_cpu();
34 75
35 pcount = per_cpu_ptr(fbc->counters, cpu); 76 preempt_disable();
36 count = *pcount + amount; 77 count = __this_cpu_read(*fbc->counters) + amount;
37 if (count >= batch || count <= -batch) { 78 if (count >= batch || count <= -batch) {
38 spin_lock(&fbc->lock); 79 spin_lock(&fbc->lock);
39 fbc->count += count; 80 fbc->count += count;
40 *pcount = 0; 81 __this_cpu_write(*fbc->counters, 0);
41 spin_unlock(&fbc->lock); 82 spin_unlock(&fbc->lock);
42 } else { 83 } else {
43 *pcount = count; 84 __this_cpu_write(*fbc->counters, count);
44 } 85 }
45 put_cpu(); 86 preempt_enable();
46} 87}
47EXPORT_SYMBOL(__percpu_counter_add); 88EXPORT_SYMBOL(__percpu_counter_add);
48 89
@@ -75,7 +116,11 @@ int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
75 fbc->counters = alloc_percpu(s32); 116 fbc->counters = alloc_percpu(s32);
76 if (!fbc->counters) 117 if (!fbc->counters)
77 return -ENOMEM; 118 return -ENOMEM;
119
120 debug_percpu_counter_activate(fbc);
121
78#ifdef CONFIG_HOTPLUG_CPU 122#ifdef CONFIG_HOTPLUG_CPU
123 INIT_LIST_HEAD(&fbc->list);
79 mutex_lock(&percpu_counters_lock); 124 mutex_lock(&percpu_counters_lock);
80 list_add(&fbc->list, &percpu_counters); 125 list_add(&fbc->list, &percpu_counters);
81 mutex_unlock(&percpu_counters_lock); 126 mutex_unlock(&percpu_counters_lock);
@@ -89,6 +134,8 @@ void percpu_counter_destroy(struct percpu_counter *fbc)
89 if (!fbc->counters) 134 if (!fbc->counters)
90 return; 135 return;
91 136
137 debug_percpu_counter_deactivate(fbc);
138
92#ifdef CONFIG_HOTPLUG_CPU 139#ifdef CONFIG_HOTPLUG_CPU
93 mutex_lock(&percpu_counters_lock); 140 mutex_lock(&percpu_counters_lock);
94 list_del(&fbc->list); 141 list_del(&fbc->list);
diff --git a/lib/plist.c b/lib/plist.c
index 1471988d9190..0ae7e6431726 100644
--- a/lib/plist.c
+++ b/lib/plist.c
@@ -28,6 +28,8 @@
28 28
29#ifdef CONFIG_DEBUG_PI_LIST 29#ifdef CONFIG_DEBUG_PI_LIST
30 30
31static struct plist_head test_head;
32
31static void plist_check_prev_next(struct list_head *t, struct list_head *p, 33static void plist_check_prev_next(struct list_head *t, struct list_head *p,
32 struct list_head *n) 34 struct list_head *n)
33{ 35{
@@ -54,12 +56,13 @@ static void plist_check_list(struct list_head *top)
54 56
55static void plist_check_head(struct plist_head *head) 57static void plist_check_head(struct plist_head *head)
56{ 58{
57 WARN_ON(!head->rawlock && !head->spinlock); 59 WARN_ON(head != &test_head && !head->rawlock && !head->spinlock);
58 if (head->rawlock) 60 if (head->rawlock)
59 WARN_ON_SMP(!raw_spin_is_locked(head->rawlock)); 61 WARN_ON_SMP(!raw_spin_is_locked(head->rawlock));
60 if (head->spinlock) 62 if (head->spinlock)
61 WARN_ON_SMP(!spin_is_locked(head->spinlock)); 63 WARN_ON_SMP(!spin_is_locked(head->spinlock));
62 plist_check_list(&head->prio_list); 64 if (!plist_head_empty(head))
65 plist_check_list(&plist_first(head)->prio_list);
63 plist_check_list(&head->node_list); 66 plist_check_list(&head->node_list);
64} 67}
65 68
@@ -75,25 +78,33 @@ static void plist_check_head(struct plist_head *head)
75 */ 78 */
76void plist_add(struct plist_node *node, struct plist_head *head) 79void plist_add(struct plist_node *node, struct plist_head *head)
77{ 80{
78 struct plist_node *iter; 81 struct plist_node *first, *iter, *prev = NULL;
82 struct list_head *node_next = &head->node_list;
79 83
80 plist_check_head(head); 84 plist_check_head(head);
81 WARN_ON(!plist_node_empty(node)); 85 WARN_ON(!plist_node_empty(node));
86 WARN_ON(!list_empty(&node->prio_list));
87
88 if (plist_head_empty(head))
89 goto ins_node;
82 90
83 list_for_each_entry(iter, &head->prio_list, plist.prio_list) { 91 first = iter = plist_first(head);
84 if (node->prio < iter->prio) 92
85 goto lt_prio; 93 do {
86 else if (node->prio == iter->prio) { 94 if (node->prio < iter->prio) {
87 iter = list_entry(iter->plist.prio_list.next, 95 node_next = &iter->node_list;
88 struct plist_node, plist.prio_list); 96 break;
89 goto eq_prio;
90 } 97 }
91 }
92 98
93lt_prio: 99 prev = iter;
94 list_add_tail(&node->plist.prio_list, &iter->plist.prio_list); 100 iter = list_entry(iter->prio_list.next,
95eq_prio: 101 struct plist_node, prio_list);
96 list_add_tail(&node->plist.node_list, &iter->plist.node_list); 102 } while (iter != first);
103
104 if (!prev || prev->prio != node->prio)
105 list_add_tail(&node->prio_list, &iter->prio_list);
106ins_node:
107 list_add_tail(&node->node_list, node_next);
97 108
98 plist_check_head(head); 109 plist_check_head(head);
99} 110}
@@ -108,14 +119,98 @@ void plist_del(struct plist_node *node, struct plist_head *head)
108{ 119{
109 plist_check_head(head); 120 plist_check_head(head);
110 121
111 if (!list_empty(&node->plist.prio_list)) { 122 if (!list_empty(&node->prio_list)) {
112 struct plist_node *next = plist_first(&node->plist); 123 if (node->node_list.next != &head->node_list) {
124 struct plist_node *next;
125
126 next = list_entry(node->node_list.next,
127 struct plist_node, node_list);
113 128
114 list_move_tail(&next->plist.prio_list, &node->plist.prio_list); 129 /* add the next plist_node into prio_list */
115 list_del_init(&node->plist.prio_list); 130 if (list_empty(&next->prio_list))
131 list_add(&next->prio_list, &node->prio_list);
132 }
133 list_del_init(&node->prio_list);
116 } 134 }
117 135
118 list_del_init(&node->plist.node_list); 136 list_del_init(&node->node_list);
119 137
120 plist_check_head(head); 138 plist_check_head(head);
121} 139}
140
141#ifdef CONFIG_DEBUG_PI_LIST
142#include <linux/sched.h>
143#include <linux/module.h>
144#include <linux/init.h>
145
146static struct plist_node __initdata test_node[241];
147
148static void __init plist_test_check(int nr_expect)
149{
150 struct plist_node *first, *prio_pos, *node_pos;
151
152 if (plist_head_empty(&test_head)) {
153 BUG_ON(nr_expect != 0);
154 return;
155 }
156
157 prio_pos = first = plist_first(&test_head);
158 plist_for_each(node_pos, &test_head) {
159 if (nr_expect-- < 0)
160 break;
161 if (node_pos == first)
162 continue;
163 if (node_pos->prio == prio_pos->prio) {
164 BUG_ON(!list_empty(&node_pos->prio_list));
165 continue;
166 }
167
168 BUG_ON(prio_pos->prio > node_pos->prio);
169 BUG_ON(prio_pos->prio_list.next != &node_pos->prio_list);
170 prio_pos = node_pos;
171 }
172
173 BUG_ON(nr_expect != 0);
174 BUG_ON(prio_pos->prio_list.next != &first->prio_list);
175}
176
177static int __init plist_test(void)
178{
179 int nr_expect = 0, i, loop;
180 unsigned int r = local_clock();
181
182 printk(KERN_INFO "start plist test\n");
183 plist_head_init(&test_head, NULL);
184 for (i = 0; i < ARRAY_SIZE(test_node); i++)
185 plist_node_init(test_node + i, 0);
186
187 for (loop = 0; loop < 1000; loop++) {
188 r = r * 193939 % 47629;
189 i = r % ARRAY_SIZE(test_node);
190 if (plist_node_empty(test_node + i)) {
191 r = r * 193939 % 47629;
192 test_node[i].prio = r % 99;
193 plist_add(test_node + i, &test_head);
194 nr_expect++;
195 } else {
196 plist_del(test_node + i, &test_head);
197 nr_expect--;
198 }
199 plist_test_check(nr_expect);
200 }
201
202 for (i = 0; i < ARRAY_SIZE(test_node); i++) {
203 if (plist_node_empty(test_node + i))
204 continue;
205 plist_del(test_node + i, &test_head);
206 nr_expect--;
207 plist_test_check(nr_expect);
208 }
209
210 printk(KERN_INFO "end plist test\n");
211 return 0;
212}
213
214module_init(plist_test);
215
216#endif
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index efd16fa80b1c..7ea2e033d715 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -49,7 +49,7 @@ struct radix_tree_node {
49 unsigned int height; /* Height from the bottom */ 49 unsigned int height; /* Height from the bottom */
50 unsigned int count; 50 unsigned int count;
51 struct rcu_head rcu_head; 51 struct rcu_head rcu_head;
52 void *slots[RADIX_TREE_MAP_SIZE]; 52 void __rcu *slots[RADIX_TREE_MAP_SIZE];
53 unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS]; 53 unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS];
54}; 54};
55 55
@@ -82,6 +82,16 @@ struct radix_tree_preload {
82}; 82};
83static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, }; 83static DEFINE_PER_CPU(struct radix_tree_preload, radix_tree_preloads) = { 0, };
84 84
85static inline void *ptr_to_indirect(void *ptr)
86{
87 return (void *)((unsigned long)ptr | RADIX_TREE_INDIRECT_PTR);
88}
89
90static inline void *indirect_to_ptr(void *ptr)
91{
92 return (void *)((unsigned long)ptr & ~RADIX_TREE_INDIRECT_PTR);
93}
94
85static inline gfp_t root_gfp_mask(struct radix_tree_root *root) 95static inline gfp_t root_gfp_mask(struct radix_tree_root *root)
86{ 96{
87 return root->gfp_mask & __GFP_BITS_MASK; 97 return root->gfp_mask & __GFP_BITS_MASK;
@@ -265,7 +275,7 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
265 return -ENOMEM; 275 return -ENOMEM;
266 276
267 /* Increase the height. */ 277 /* Increase the height. */
268 node->slots[0] = radix_tree_indirect_to_ptr(root->rnode); 278 node->slots[0] = indirect_to_ptr(root->rnode);
269 279
270 /* Propagate the aggregated tag info into the new root */ 280 /* Propagate the aggregated tag info into the new root */
271 for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) { 281 for (tag = 0; tag < RADIX_TREE_MAX_TAGS; tag++) {
@@ -276,7 +286,7 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
276 newheight = root->height+1; 286 newheight = root->height+1;
277 node->height = newheight; 287 node->height = newheight;
278 node->count = 1; 288 node->count = 1;
279 node = radix_tree_ptr_to_indirect(node); 289 node = ptr_to_indirect(node);
280 rcu_assign_pointer(root->rnode, node); 290 rcu_assign_pointer(root->rnode, node);
281 root->height = newheight; 291 root->height = newheight;
282 } while (height > root->height); 292 } while (height > root->height);
@@ -309,7 +319,7 @@ int radix_tree_insert(struct radix_tree_root *root,
309 return error; 319 return error;
310 } 320 }
311 321
312 slot = radix_tree_indirect_to_ptr(root->rnode); 322 slot = indirect_to_ptr(root->rnode);
313 323
314 height = root->height; 324 height = root->height;
315 shift = (height-1) * RADIX_TREE_MAP_SHIFT; 325 shift = (height-1) * RADIX_TREE_MAP_SHIFT;
@@ -325,8 +335,7 @@ int radix_tree_insert(struct radix_tree_root *root,
325 rcu_assign_pointer(node->slots[offset], slot); 335 rcu_assign_pointer(node->slots[offset], slot);
326 node->count++; 336 node->count++;
327 } else 337 } else
328 rcu_assign_pointer(root->rnode, 338 rcu_assign_pointer(root->rnode, ptr_to_indirect(slot));
329 radix_tree_ptr_to_indirect(slot));
330 } 339 }
331 340
332 /* Go a level down */ 341 /* Go a level down */
@@ -374,7 +383,7 @@ static void *radix_tree_lookup_element(struct radix_tree_root *root,
374 return NULL; 383 return NULL;
375 return is_slot ? (void *)&root->rnode : node; 384 return is_slot ? (void *)&root->rnode : node;
376 } 385 }
377 node = radix_tree_indirect_to_ptr(node); 386 node = indirect_to_ptr(node);
378 387
379 height = node->height; 388 height = node->height;
380 if (index > radix_tree_maxindex(height)) 389 if (index > radix_tree_maxindex(height))
@@ -393,7 +402,7 @@ static void *radix_tree_lookup_element(struct radix_tree_root *root,
393 height--; 402 height--;
394 } while (height > 0); 403 } while (height > 0);
395 404
396 return is_slot ? (void *)slot:node; 405 return is_slot ? (void *)slot : indirect_to_ptr(node);
397} 406}
398 407
399/** 408/**
@@ -455,7 +464,7 @@ void *radix_tree_tag_set(struct radix_tree_root *root,
455 height = root->height; 464 height = root->height;
456 BUG_ON(index > radix_tree_maxindex(height)); 465 BUG_ON(index > radix_tree_maxindex(height));
457 466
458 slot = radix_tree_indirect_to_ptr(root->rnode); 467 slot = indirect_to_ptr(root->rnode);
459 shift = (height - 1) * RADIX_TREE_MAP_SHIFT; 468 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
460 469
461 while (height > 0) { 470 while (height > 0) {
@@ -509,7 +518,7 @@ void *radix_tree_tag_clear(struct radix_tree_root *root,
509 518
510 shift = (height - 1) * RADIX_TREE_MAP_SHIFT; 519 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
511 pathp->node = NULL; 520 pathp->node = NULL;
512 slot = radix_tree_indirect_to_ptr(root->rnode); 521 slot = indirect_to_ptr(root->rnode);
513 522
514 while (height > 0) { 523 while (height > 0) {
515 int offset; 524 int offset;
@@ -579,7 +588,7 @@ int radix_tree_tag_get(struct radix_tree_root *root,
579 588
580 if (!radix_tree_is_indirect_ptr(node)) 589 if (!radix_tree_is_indirect_ptr(node))
581 return (index == 0); 590 return (index == 0);
582 node = radix_tree_indirect_to_ptr(node); 591 node = indirect_to_ptr(node);
583 592
584 height = node->height; 593 height = node->height;
585 if (index > radix_tree_maxindex(height)) 594 if (index > radix_tree_maxindex(height))
@@ -666,7 +675,7 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
666 } 675 }
667 676
668 shift = (height - 1) * RADIX_TREE_MAP_SHIFT; 677 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
669 slot = radix_tree_indirect_to_ptr(root->rnode); 678 slot = indirect_to_ptr(root->rnode);
670 679
671 /* 680 /*
672 * we fill the path from (root->height - 2) to 0, leaving the index at 681 * we fill the path from (root->height - 2) to 0, leaving the index at
@@ -727,10 +736,11 @@ next:
727 } 736 }
728 } 737 }
729 /* 738 /*
730 * The iftag must have been set somewhere because otherwise 739 * We need not to tag the root tag if there is no tag which is set with
731 * we would return immediated at the beginning of the function 740 * settag within the range from *first_indexp to last_index.
732 */ 741 */
733 root_tag_set(root, settag); 742 if (tagged > 0)
743 root_tag_set(root, settag);
734 *first_indexp = index; 744 *first_indexp = index;
735 745
736 return tagged; 746 return tagged;
@@ -897,7 +907,7 @@ radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
897 results[0] = node; 907 results[0] = node;
898 return 1; 908 return 1;
899 } 909 }
900 node = radix_tree_indirect_to_ptr(node); 910 node = indirect_to_ptr(node);
901 911
902 max_index = radix_tree_maxindex(node->height); 912 max_index = radix_tree_maxindex(node->height);
903 913
@@ -916,7 +926,8 @@ radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
916 slot = *(((void ***)results)[ret + i]); 926 slot = *(((void ***)results)[ret + i]);
917 if (!slot) 927 if (!slot)
918 continue; 928 continue;
919 results[ret + nr_found] = rcu_dereference_raw(slot); 929 results[ret + nr_found] =
930 indirect_to_ptr(rcu_dereference_raw(slot));
920 nr_found++; 931 nr_found++;
921 } 932 }
922 ret += nr_found; 933 ret += nr_found;
@@ -965,7 +976,7 @@ radix_tree_gang_lookup_slot(struct radix_tree_root *root, void ***results,
965 results[0] = (void **)&root->rnode; 976 results[0] = (void **)&root->rnode;
966 return 1; 977 return 1;
967 } 978 }
968 node = radix_tree_indirect_to_ptr(node); 979 node = indirect_to_ptr(node);
969 980
970 max_index = radix_tree_maxindex(node->height); 981 max_index = radix_tree_maxindex(node->height);
971 982
@@ -1090,7 +1101,7 @@ radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
1090 results[0] = node; 1101 results[0] = node;
1091 return 1; 1102 return 1;
1092 } 1103 }
1093 node = radix_tree_indirect_to_ptr(node); 1104 node = indirect_to_ptr(node);
1094 1105
1095 max_index = radix_tree_maxindex(node->height); 1106 max_index = radix_tree_maxindex(node->height);
1096 1107
@@ -1109,7 +1120,8 @@ radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
1109 slot = *(((void ***)results)[ret + i]); 1120 slot = *(((void ***)results)[ret + i]);
1110 if (!slot) 1121 if (!slot)
1111 continue; 1122 continue;
1112 results[ret + nr_found] = rcu_dereference_raw(slot); 1123 results[ret + nr_found] =
1124 indirect_to_ptr(rcu_dereference_raw(slot));
1113 nr_found++; 1125 nr_found++;
1114 } 1126 }
1115 ret += nr_found; 1127 ret += nr_found;
@@ -1159,7 +1171,7 @@ radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
1159 results[0] = (void **)&root->rnode; 1171 results[0] = (void **)&root->rnode;
1160 return 1; 1172 return 1;
1161 } 1173 }
1162 node = radix_tree_indirect_to_ptr(node); 1174 node = indirect_to_ptr(node);
1163 1175
1164 max_index = radix_tree_maxindex(node->height); 1176 max_index = radix_tree_maxindex(node->height);
1165 1177
@@ -1195,7 +1207,7 @@ static inline void radix_tree_shrink(struct radix_tree_root *root)
1195 void *newptr; 1207 void *newptr;
1196 1208
1197 BUG_ON(!radix_tree_is_indirect_ptr(to_free)); 1209 BUG_ON(!radix_tree_is_indirect_ptr(to_free));
1198 to_free = radix_tree_indirect_to_ptr(to_free); 1210 to_free = indirect_to_ptr(to_free);
1199 1211
1200 /* 1212 /*
1201 * The candidate node has more than one child, or its child 1213 * The candidate node has more than one child, or its child
@@ -1208,16 +1220,39 @@ static inline void radix_tree_shrink(struct radix_tree_root *root)
1208 1220
1209 /* 1221 /*
1210 * We don't need rcu_assign_pointer(), since we are simply 1222 * We don't need rcu_assign_pointer(), since we are simply
1211 * moving the node from one part of the tree to another. If 1223 * moving the node from one part of the tree to another: if it
1212 * it was safe to dereference the old pointer to it 1224 * was safe to dereference the old pointer to it
1213 * (to_free->slots[0]), it will be safe to dereference the new 1225 * (to_free->slots[0]), it will be safe to dereference the new
1214 * one (root->rnode). 1226 * one (root->rnode) as far as dependent read barriers go.
1215 */ 1227 */
1216 newptr = to_free->slots[0]; 1228 newptr = to_free->slots[0];
1217 if (root->height > 1) 1229 if (root->height > 1)
1218 newptr = radix_tree_ptr_to_indirect(newptr); 1230 newptr = ptr_to_indirect(newptr);
1219 root->rnode = newptr; 1231 root->rnode = newptr;
1220 root->height--; 1232 root->height--;
1233
1234 /*
1235 * We have a dilemma here. The node's slot[0] must not be
1236 * NULLed in case there are concurrent lookups expecting to
1237 * find the item. However if this was a bottom-level node,
1238 * then it may be subject to the slot pointer being visible
1239 * to callers dereferencing it. If item corresponding to
1240 * slot[0] is subsequently deleted, these callers would expect
1241 * their slot to become empty sooner or later.
1242 *
1243 * For example, lockless pagecache will look up a slot, deref
1244 * the page pointer, and if the page is 0 refcount it means it
1245 * was concurrently deleted from pagecache so try the deref
1246 * again. Fortunately there is already a requirement for logic
1247 * to retry the entire slot lookup -- the indirect pointer
1248 * problem (replacing direct root node with an indirect pointer
1249 * also results in a stale slot). So tag the slot as indirect
1250 * to force callers to retry.
1251 */
1252 if (root->height == 0)
1253 *((unsigned long *)&to_free->slots[0]) |=
1254 RADIX_TREE_INDIRECT_PTR;
1255
1221 radix_tree_node_free(to_free); 1256 radix_tree_node_free(to_free);
1222 } 1257 }
1223} 1258}
@@ -1254,7 +1289,7 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
1254 root->rnode = NULL; 1289 root->rnode = NULL;
1255 goto out; 1290 goto out;
1256 } 1291 }
1257 slot = radix_tree_indirect_to_ptr(slot); 1292 slot = indirect_to_ptr(slot);
1258 1293
1259 shift = (height - 1) * RADIX_TREE_MAP_SHIFT; 1294 shift = (height - 1) * RADIX_TREE_MAP_SHIFT;
1260 pathp->node = NULL; 1295 pathp->node = NULL;
@@ -1296,8 +1331,7 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
1296 radix_tree_node_free(to_free); 1331 radix_tree_node_free(to_free);
1297 1332
1298 if (pathp->node->count) { 1333 if (pathp->node->count) {
1299 if (pathp->node == 1334 if (pathp->node == indirect_to_ptr(root->rnode))
1300 radix_tree_indirect_to_ptr(root->rnode))
1301 radix_tree_shrink(root); 1335 radix_tree_shrink(root);
1302 goto out; 1336 goto out;
1303 } 1337 }
diff --git a/lib/rbtree.c b/lib/rbtree.c
index 4693f79195d3..a16be19a1305 100644
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -315,6 +315,7 @@ void rb_augment_insert(struct rb_node *node, rb_augment_f func, void *data)
315 315
316 rb_augment_path(node, func, data); 316 rb_augment_path(node, func, data);
317} 317}
318EXPORT_SYMBOL(rb_augment_insert);
318 319
319/* 320/*
320 * before removing the node, find the deepest node on the rebalance path 321 * before removing the node, find the deepest node on the rebalance path
@@ -340,6 +341,7 @@ struct rb_node *rb_augment_erase_begin(struct rb_node *node)
340 341
341 return deepest; 342 return deepest;
342} 343}
344EXPORT_SYMBOL(rb_augment_erase_begin);
343 345
344/* 346/*
345 * after removal, update the tree to account for the removed entry 347 * after removal, update the tree to account for the removed entry
@@ -350,6 +352,7 @@ void rb_augment_erase_end(struct rb_node *node, rb_augment_f func, void *data)
350 if (node) 352 if (node)
351 rb_augment_path(node, func, data); 353 rb_augment_path(node, func, data);
352} 354}
355EXPORT_SYMBOL(rb_augment_erase_end);
353 356
354/* 357/*
355 * This function returns the first node (in sort order) of the tree. 358 * This function returns the first node (in sort order) of the tree.
diff --git a/lib/rwsem.c b/lib/rwsem.c
index f236d7cd5cf3..aa7c3052261f 100644
--- a/lib/rwsem.c
+++ b/lib/rwsem.c
@@ -222,8 +222,7 @@ rwsem_down_failed_common(struct rw_semaphore *sem,
222/* 222/*
223 * wait for the read lock to be granted 223 * wait for the read lock to be granted
224 */ 224 */
225asmregparm struct rw_semaphore __sched * 225struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
226rwsem_down_read_failed(struct rw_semaphore *sem)
227{ 226{
228 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_READ, 227 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_READ,
229 -RWSEM_ACTIVE_READ_BIAS); 228 -RWSEM_ACTIVE_READ_BIAS);
@@ -232,8 +231,7 @@ rwsem_down_read_failed(struct rw_semaphore *sem)
232/* 231/*
233 * wait for the write lock to be granted 232 * wait for the write lock to be granted
234 */ 233 */
235asmregparm struct rw_semaphore __sched * 234struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
236rwsem_down_write_failed(struct rw_semaphore *sem)
237{ 235{
238 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_WRITE, 236 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_WRITE,
239 -RWSEM_ACTIVE_WRITE_BIAS); 237 -RWSEM_ACTIVE_WRITE_BIAS);
@@ -243,7 +241,7 @@ rwsem_down_write_failed(struct rw_semaphore *sem)
243 * handle waking up a waiter on the semaphore 241 * handle waking up a waiter on the semaphore
244 * - up_read/up_write has decremented the active part of count if we come here 242 * - up_read/up_write has decremented the active part of count if we come here
245 */ 243 */
246asmregparm struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem) 244struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)
247{ 245{
248 unsigned long flags; 246 unsigned long flags;
249 247
@@ -263,7 +261,7 @@ asmregparm struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)
263 * - caller incremented waiting part of count and discovered it still negative 261 * - caller incremented waiting part of count and discovered it still negative
264 * - just wake up any readers at the front of the queue 262 * - just wake up any readers at the front of the queue
265 */ 263 */
266asmregparm struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) 264struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)
267{ 265{
268 unsigned long flags; 266 unsigned long flags;
269 267
diff --git a/lib/show_mem.c b/lib/show_mem.c
index fdc77c82f922..4407f8c9b1f7 100644
--- a/lib/show_mem.c
+++ b/lib/show_mem.c
@@ -9,14 +9,14 @@
9#include <linux/nmi.h> 9#include <linux/nmi.h>
10#include <linux/quicklist.h> 10#include <linux/quicklist.h>
11 11
12void show_mem(void) 12void show_mem(unsigned int filter)
13{ 13{
14 pg_data_t *pgdat; 14 pg_data_t *pgdat;
15 unsigned long total = 0, reserved = 0, shared = 0, 15 unsigned long total = 0, reserved = 0, shared = 0,
16 nonshared = 0, highmem = 0; 16 nonshared = 0, highmem = 0;
17 17
18 printk("Mem-Info:\n"); 18 printk("Mem-Info:\n");
19 show_free_areas(); 19 show_free_areas(filter);
20 20
21 for_each_online_pgdat(pgdat) { 21 for_each_online_pgdat(pgdat) {
22 unsigned long i, flags; 22 unsigned long i, flags;
diff --git a/lib/string.c b/lib/string.c
index f71bead1be3e..01fad9b203e1 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -535,6 +535,35 @@ bool sysfs_streq(const char *s1, const char *s2)
535} 535}
536EXPORT_SYMBOL(sysfs_streq); 536EXPORT_SYMBOL(sysfs_streq);
537 537
538/**
539 * strtobool - convert common user inputs into boolean values
540 * @s: input string
541 * @res: result
542 *
543 * This routine returns 0 iff the first character is one of 'Yy1Nn0'.
544 * Otherwise it will return -EINVAL. Value pointed to by res is
545 * updated upon finding a match.
546 */
547int strtobool(const char *s, bool *res)
548{
549 switch (s[0]) {
550 case 'y':
551 case 'Y':
552 case '1':
553 *res = true;
554 break;
555 case 'n':
556 case 'N':
557 case '0':
558 *res = false;
559 break;
560 default:
561 return -EINVAL;
562 }
563 return 0;
564}
565EXPORT_SYMBOL(strtobool);
566
538#ifndef __HAVE_ARCH_MEMSET 567#ifndef __HAVE_ARCH_MEMSET
539/** 568/**
540 * memset - Fill a region of memory with the given value 569 * memset - Fill a region of memory with the given value
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 34e3082632d8..99093b396145 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -60,7 +60,7 @@ int swiotlb_force;
60static char *io_tlb_start, *io_tlb_end; 60static char *io_tlb_start, *io_tlb_end;
61 61
62/* 62/*
63 * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and 63 * The number of IO TLB blocks (in groups of 64) between io_tlb_start and
64 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages. 64 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
65 */ 65 */
66static unsigned long io_tlb_nslabs; 66static unsigned long io_tlb_nslabs;
@@ -70,7 +70,7 @@ static unsigned long io_tlb_nslabs;
70 */ 70 */
71static unsigned long io_tlb_overflow = 32*1024; 71static unsigned long io_tlb_overflow = 32*1024;
72 72
73void *io_tlb_overflow_buffer; 73static void *io_tlb_overflow_buffer;
74 74
75/* 75/*
76 * This is a free list describing the number of free entries available from 76 * This is a free list describing the number of free entries available from
@@ -110,6 +110,11 @@ setup_io_tlb_npages(char *str)
110__setup("swiotlb=", setup_io_tlb_npages); 110__setup("swiotlb=", setup_io_tlb_npages);
111/* make io_tlb_overflow tunable too? */ 111/* make io_tlb_overflow tunable too? */
112 112
113unsigned long swioltb_nr_tbl(void)
114{
115 return io_tlb_nslabs;
116}
117
113/* Note that this doesn't work with highmem page */ 118/* Note that this doesn't work with highmem page */
114static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev, 119static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
115 volatile void *address) 120 volatile void *address)
@@ -147,16 +152,16 @@ void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
147 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE 152 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
148 * between io_tlb_start and io_tlb_end. 153 * between io_tlb_start and io_tlb_end.
149 */ 154 */
150 io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int)); 155 io_tlb_list = alloc_bootmem_pages(PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));
151 for (i = 0; i < io_tlb_nslabs; i++) 156 for (i = 0; i < io_tlb_nslabs; i++)
152 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); 157 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
153 io_tlb_index = 0; 158 io_tlb_index = 0;
154 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(phys_addr_t)); 159 io_tlb_orig_addr = alloc_bootmem_pages(PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
155 160
156 /* 161 /*
157 * Get the overflow emergency buffer 162 * Get the overflow emergency buffer
158 */ 163 */
159 io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow); 164 io_tlb_overflow_buffer = alloc_bootmem_low_pages(PAGE_ALIGN(io_tlb_overflow));
160 if (!io_tlb_overflow_buffer) 165 if (!io_tlb_overflow_buffer)
161 panic("Cannot allocate SWIOTLB overflow buffer!\n"); 166 panic("Cannot allocate SWIOTLB overflow buffer!\n");
162 if (verbose) 167 if (verbose)
@@ -182,7 +187,7 @@ swiotlb_init_with_default_size(size_t default_size, int verbose)
182 /* 187 /*
183 * Get IO TLB memory from the low pages 188 * Get IO TLB memory from the low pages
184 */ 189 */
185 io_tlb_start = alloc_bootmem_low_pages(bytes); 190 io_tlb_start = alloc_bootmem_low_pages(PAGE_ALIGN(bytes));
186 if (!io_tlb_start) 191 if (!io_tlb_start)
187 panic("Cannot allocate SWIOTLB buffer"); 192 panic("Cannot allocate SWIOTLB buffer");
188 193
@@ -308,13 +313,13 @@ void __init swiotlb_free(void)
308 get_order(io_tlb_nslabs << IO_TLB_SHIFT)); 313 get_order(io_tlb_nslabs << IO_TLB_SHIFT));
309 } else { 314 } else {
310 free_bootmem_late(__pa(io_tlb_overflow_buffer), 315 free_bootmem_late(__pa(io_tlb_overflow_buffer),
311 io_tlb_overflow); 316 PAGE_ALIGN(io_tlb_overflow));
312 free_bootmem_late(__pa(io_tlb_orig_addr), 317 free_bootmem_late(__pa(io_tlb_orig_addr),
313 io_tlb_nslabs * sizeof(phys_addr_t)); 318 PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
314 free_bootmem_late(__pa(io_tlb_list), 319 free_bootmem_late(__pa(io_tlb_list),
315 io_tlb_nslabs * sizeof(int)); 320 PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));
316 free_bootmem_late(__pa(io_tlb_start), 321 free_bootmem_late(__pa(io_tlb_start),
317 io_tlb_nslabs << IO_TLB_SHIFT); 322 PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));
318 } 323 }
319} 324}
320 325
@@ -686,8 +691,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
686 /* 691 /*
687 * Ensure that the address returned is DMA'ble 692 * Ensure that the address returned is DMA'ble
688 */ 693 */
689 if (!dma_capable(dev, dev_addr, size)) 694 if (!dma_capable(dev, dev_addr, size)) {
690 panic("map_single: bounce buffer is not DMA'ble"); 695 swiotlb_tbl_unmap_single(dev, map, size, dir);
696 dev_addr = swiotlb_virt_to_bus(dev, io_tlb_overflow_buffer);
697 }
691 698
692 return dev_addr; 699 return dev_addr;
693} 700}
diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c
new file mode 100644
index 000000000000..d55769d63cb8
--- /dev/null
+++ b/lib/test-kstrtox.c
@@ -0,0 +1,739 @@
1#include <linux/init.h>
2#include <linux/kernel.h>
3#include <linux/module.h>
4
5#define for_each_test(i, test) \
6 for (i = 0; i < sizeof(test) / sizeof(test[0]); i++)
7
8struct test_fail {
9 const char *str;
10 unsigned int base;
11};
12
13#define DEFINE_TEST_FAIL(test) \
14 const struct test_fail test[] __initdata
15
16#define DECLARE_TEST_OK(type, test_type) \
17 test_type { \
18 const char *str; \
19 unsigned int base; \
20 type expected_res; \
21 }
22
23#define DEFINE_TEST_OK(type, test) \
24 const type test[] __initdata
25
26#define TEST_FAIL(fn, type, fmt, test) \
27{ \
28 unsigned int i; \
29 \
30 for_each_test(i, test) { \
31 const struct test_fail *t = &test[i]; \
32 type tmp; \
33 int rv; \
34 \
35 tmp = 0; \
36 rv = fn(t->str, t->base, &tmp); \
37 if (rv >= 0) { \
38 WARN(1, "str '%s', base %u, expected -E, got %d/" fmt "\n", \
39 t->str, t->base, rv, tmp); \
40 continue; \
41 } \
42 } \
43}
44
45#define TEST_OK(fn, type, fmt, test) \
46{ \
47 unsigned int i; \
48 \
49 for_each_test(i, test) { \
50 const typeof(test[0]) *t = &test[i]; \
51 type res; \
52 int rv; \
53 \
54 rv = fn(t->str, t->base, &res); \
55 if (rv != 0) { \
56 WARN(1, "str '%s', base %u, expected 0/" fmt ", got %d\n", \
57 t->str, t->base, t->expected_res, rv); \
58 continue; \
59 } \
60 if (res != t->expected_res) { \
61 WARN(1, "str '%s', base %u, expected " fmt ", got " fmt "\n", \
62 t->str, t->base, t->expected_res, res); \
63 continue; \
64 } \
65 } \
66}
67
68static void __init test_kstrtoull_ok(void)
69{
70 DECLARE_TEST_OK(unsigned long long, struct test_ull);
71 static DEFINE_TEST_OK(struct test_ull, test_ull_ok) = {
72 {"0", 10, 0ULL},
73 {"1", 10, 1ULL},
74 {"127", 10, 127ULL},
75 {"128", 10, 128ULL},
76 {"129", 10, 129ULL},
77 {"255", 10, 255ULL},
78 {"256", 10, 256ULL},
79 {"257", 10, 257ULL},
80 {"32767", 10, 32767ULL},
81 {"32768", 10, 32768ULL},
82 {"32769", 10, 32769ULL},
83 {"65535", 10, 65535ULL},
84 {"65536", 10, 65536ULL},
85 {"65537", 10, 65537ULL},
86 {"2147483647", 10, 2147483647ULL},
87 {"2147483648", 10, 2147483648ULL},
88 {"2147483649", 10, 2147483649ULL},
89 {"4294967295", 10, 4294967295ULL},
90 {"4294967296", 10, 4294967296ULL},
91 {"4294967297", 10, 4294967297ULL},
92 {"9223372036854775807", 10, 9223372036854775807ULL},
93 {"9223372036854775808", 10, 9223372036854775808ULL},
94 {"9223372036854775809", 10, 9223372036854775809ULL},
95 {"18446744073709551614", 10, 18446744073709551614ULL},
96 {"18446744073709551615", 10, 18446744073709551615ULL},
97
98 {"00", 8, 00ULL},
99 {"01", 8, 01ULL},
100 {"0177", 8, 0177ULL},
101 {"0200", 8, 0200ULL},
102 {"0201", 8, 0201ULL},
103 {"0377", 8, 0377ULL},
104 {"0400", 8, 0400ULL},
105 {"0401", 8, 0401ULL},
106 {"077777", 8, 077777ULL},
107 {"0100000", 8, 0100000ULL},
108 {"0100001", 8, 0100001ULL},
109 {"0177777", 8, 0177777ULL},
110 {"0200000", 8, 0200000ULL},
111 {"0200001", 8, 0200001ULL},
112 {"017777777777", 8, 017777777777ULL},
113 {"020000000000", 8, 020000000000ULL},
114 {"020000000001", 8, 020000000001ULL},
115 {"037777777777", 8, 037777777777ULL},
116 {"040000000000", 8, 040000000000ULL},
117 {"040000000001", 8, 040000000001ULL},
118 {"0777777777777777777777", 8, 0777777777777777777777ULL},
119 {"01000000000000000000000", 8, 01000000000000000000000ULL},
120 {"01000000000000000000001", 8, 01000000000000000000001ULL},
121 {"01777777777777777777776", 8, 01777777777777777777776ULL},
122 {"01777777777777777777777", 8, 01777777777777777777777ULL},
123
124 {"0x0", 16, 0x0ULL},
125 {"0x1", 16, 0x1ULL},
126 {"0x7f", 16, 0x7fULL},
127 {"0x80", 16, 0x80ULL},
128 {"0x81", 16, 0x81ULL},
129 {"0xff", 16, 0xffULL},
130 {"0x100", 16, 0x100ULL},
131 {"0x101", 16, 0x101ULL},
132 {"0x7fff", 16, 0x7fffULL},
133 {"0x8000", 16, 0x8000ULL},
134 {"0x8001", 16, 0x8001ULL},
135 {"0xffff", 16, 0xffffULL},
136 {"0x10000", 16, 0x10000ULL},
137 {"0x10001", 16, 0x10001ULL},
138 {"0x7fffffff", 16, 0x7fffffffULL},
139 {"0x80000000", 16, 0x80000000ULL},
140 {"0x80000001", 16, 0x80000001ULL},
141 {"0xffffffff", 16, 0xffffffffULL},
142 {"0x100000000", 16, 0x100000000ULL},
143 {"0x100000001", 16, 0x100000001ULL},
144 {"0x7fffffffffffffff", 16, 0x7fffffffffffffffULL},
145 {"0x8000000000000000", 16, 0x8000000000000000ULL},
146 {"0x8000000000000001", 16, 0x8000000000000001ULL},
147 {"0xfffffffffffffffe", 16, 0xfffffffffffffffeULL},
148 {"0xffffffffffffffff", 16, 0xffffffffffffffffULL},
149
150 {"0\n", 0, 0ULL},
151 };
152 TEST_OK(kstrtoull, unsigned long long, "%llu", test_ull_ok);
153}
154
155static void __init test_kstrtoull_fail(void)
156{
157 static DEFINE_TEST_FAIL(test_ull_fail) = {
158 {"", 0},
159 {"", 8},
160 {"", 10},
161 {"", 16},
162 {"\n", 0},
163 {"\n", 8},
164 {"\n", 10},
165 {"\n", 16},
166 {"\n0", 0},
167 {"\n0", 8},
168 {"\n0", 10},
169 {"\n0", 16},
170 {"+", 0},
171 {"+", 8},
172 {"+", 10},
173 {"+", 16},
174 {"-", 0},
175 {"-", 8},
176 {"-", 10},
177 {"-", 16},
178 {"0x", 0},
179 {"0x", 16},
180 {"0X", 0},
181 {"0X", 16},
182 {"0 ", 0},
183 {"1+", 0},
184 {"1-", 0},
185 {" 2", 0},
186 /* base autodetection */
187 {"0x0z", 0},
188 {"0z", 0},
189 {"a", 0},
190 /* digit >= base */
191 {"2", 2},
192 {"8", 8},
193 {"a", 10},
194 {"A", 10},
195 {"g", 16},
196 {"G", 16},
197 /* overflow */
198 {"10000000000000000000000000000000000000000000000000000000000000000", 2},
199 {"2000000000000000000000", 8},
200 {"18446744073709551616", 10},
201 {"10000000000000000", 16},
202 /* negative */
203 {"-0", 0},
204 {"-0", 8},
205 {"-0", 10},
206 {"-0", 16},
207 {"-1", 0},
208 {"-1", 8},
209 {"-1", 10},
210 {"-1", 16},
211 /* sign is first character if any */
212 {"-+1", 0},
213 {"-+1", 8},
214 {"-+1", 10},
215 {"-+1", 16},
216 /* nothing after \n */
217 {"0\n0", 0},
218 {"0\n0", 8},
219 {"0\n0", 10},
220 {"0\n0", 16},
221 {"0\n+", 0},
222 {"0\n+", 8},
223 {"0\n+", 10},
224 {"0\n+", 16},
225 {"0\n-", 0},
226 {"0\n-", 8},
227 {"0\n-", 10},
228 {"0\n-", 16},
229 {"0\n ", 0},
230 {"0\n ", 8},
231 {"0\n ", 10},
232 {"0\n ", 16},
233 };
234 TEST_FAIL(kstrtoull, unsigned long long, "%llu", test_ull_fail);
235}
236
237static void __init test_kstrtoll_ok(void)
238{
239 DECLARE_TEST_OK(long long, struct test_ll);
240 static DEFINE_TEST_OK(struct test_ll, test_ll_ok) = {
241 {"0", 10, 0LL},
242 {"1", 10, 1LL},
243 {"127", 10, 127LL},
244 {"128", 10, 128LL},
245 {"129", 10, 129LL},
246 {"255", 10, 255LL},
247 {"256", 10, 256LL},
248 {"257", 10, 257LL},
249 {"32767", 10, 32767LL},
250 {"32768", 10, 32768LL},
251 {"32769", 10, 32769LL},
252 {"65535", 10, 65535LL},
253 {"65536", 10, 65536LL},
254 {"65537", 10, 65537LL},
255 {"2147483647", 10, 2147483647LL},
256 {"2147483648", 10, 2147483648LL},
257 {"2147483649", 10, 2147483649LL},
258 {"4294967295", 10, 4294967295LL},
259 {"4294967296", 10, 4294967296LL},
260 {"4294967297", 10, 4294967297LL},
261 {"9223372036854775807", 10, 9223372036854775807LL},
262
263 {"-1", 10, -1LL},
264 {"-2", 10, -2LL},
265 {"-9223372036854775808", 10, LLONG_MIN},
266 };
267 TEST_OK(kstrtoll, long long, "%lld", test_ll_ok);
268}
269
270static void __init test_kstrtoll_fail(void)
271{
272 static DEFINE_TEST_FAIL(test_ll_fail) = {
273 {"9223372036854775808", 10},
274 {"9223372036854775809", 10},
275 {"18446744073709551614", 10},
276 {"18446744073709551615", 10},
277 {"-9223372036854775809", 10},
278 {"-18446744073709551614", 10},
279 {"-18446744073709551615", 10},
280 /* negative zero isn't an integer in Linux */
281 {"-0", 0},
282 {"-0", 8},
283 {"-0", 10},
284 {"-0", 16},
285 /* sign is first character if any */
286 {"-+1", 0},
287 {"-+1", 8},
288 {"-+1", 10},
289 {"-+1", 16},
290 };
291 TEST_FAIL(kstrtoll, long long, "%lld", test_ll_fail);
292}
293
294static void __init test_kstrtou64_ok(void)
295{
296 DECLARE_TEST_OK(u64, struct test_u64);
297 static DEFINE_TEST_OK(struct test_u64, test_u64_ok) = {
298 {"0", 10, 0},
299 {"1", 10, 1},
300 {"126", 10, 126},
301 {"127", 10, 127},
302 {"128", 10, 128},
303 {"129", 10, 129},
304 {"254", 10, 254},
305 {"255", 10, 255},
306 {"256", 10, 256},
307 {"257", 10, 257},
308 {"32766", 10, 32766},
309 {"32767", 10, 32767},
310 {"32768", 10, 32768},
311 {"32769", 10, 32769},
312 {"65534", 10, 65534},
313 {"65535", 10, 65535},
314 {"65536", 10, 65536},
315 {"65537", 10, 65537},
316 {"2147483646", 10, 2147483646},
317 {"2147483647", 10, 2147483647},
318 {"2147483648", 10, 2147483648ULL},
319 {"2147483649", 10, 2147483649ULL},
320 {"4294967294", 10, 4294967294ULL},
321 {"4294967295", 10, 4294967295ULL},
322 {"4294967296", 10, 4294967296ULL},
323 {"4294967297", 10, 4294967297ULL},
324 {"9223372036854775806", 10, 9223372036854775806ULL},
325 {"9223372036854775807", 10, 9223372036854775807ULL},
326 {"9223372036854775808", 10, 9223372036854775808ULL},
327 {"9223372036854775809", 10, 9223372036854775809ULL},
328 {"18446744073709551614", 10, 18446744073709551614ULL},
329 {"18446744073709551615", 10, 18446744073709551615ULL},
330 };
331 TEST_OK(kstrtou64, u64, "%llu", test_u64_ok);
332}
333
334static void __init test_kstrtou64_fail(void)
335{
336 static DEFINE_TEST_FAIL(test_u64_fail) = {
337 {"-2", 10},
338 {"-1", 10},
339 {"18446744073709551616", 10},
340 {"18446744073709551617", 10},
341 };
342 TEST_FAIL(kstrtou64, u64, "%llu", test_u64_fail);
343}
344
345static void __init test_kstrtos64_ok(void)
346{
347 DECLARE_TEST_OK(s64, struct test_s64);
348 static DEFINE_TEST_OK(struct test_s64, test_s64_ok) = {
349 {"-128", 10, -128},
350 {"-127", 10, -127},
351 {"-1", 10, -1},
352 {"0", 10, 0},
353 {"1", 10, 1},
354 {"126", 10, 126},
355 {"127", 10, 127},
356 {"128", 10, 128},
357 {"129", 10, 129},
358 {"254", 10, 254},
359 {"255", 10, 255},
360 {"256", 10, 256},
361 {"257", 10, 257},
362 {"32766", 10, 32766},
363 {"32767", 10, 32767},
364 {"32768", 10, 32768},
365 {"32769", 10, 32769},
366 {"65534", 10, 65534},
367 {"65535", 10, 65535},
368 {"65536", 10, 65536},
369 {"65537", 10, 65537},
370 {"2147483646", 10, 2147483646},
371 {"2147483647", 10, 2147483647},
372 {"2147483648", 10, 2147483648LL},
373 {"2147483649", 10, 2147483649LL},
374 {"4294967294", 10, 4294967294LL},
375 {"4294967295", 10, 4294967295LL},
376 {"4294967296", 10, 4294967296LL},
377 {"4294967297", 10, 4294967297LL},
378 {"9223372036854775806", 10, 9223372036854775806LL},
379 {"9223372036854775807", 10, 9223372036854775807LL},
380 };
381 TEST_OK(kstrtos64, s64, "%lld", test_s64_ok);
382}
383
384static void __init test_kstrtos64_fail(void)
385{
386 static DEFINE_TEST_FAIL(test_s64_fail) = {
387 {"9223372036854775808", 10},
388 {"9223372036854775809", 10},
389 {"18446744073709551614", 10},
390 {"18446744073709551615", 10},
391 {"18446744073709551616", 10},
392 {"18446744073709551617", 10},
393 };
394 TEST_FAIL(kstrtos64, s64, "%lld", test_s64_fail);
395}
396
397static void __init test_kstrtou32_ok(void)
398{
399 DECLARE_TEST_OK(u32, struct test_u32);
400 static DEFINE_TEST_OK(struct test_u32, test_u32_ok) = {
401 {"0", 10, 0},
402 {"1", 10, 1},
403 {"126", 10, 126},
404 {"127", 10, 127},
405 {"128", 10, 128},
406 {"129", 10, 129},
407 {"254", 10, 254},
408 {"255", 10, 255},
409 {"256", 10, 256},
410 {"257", 10, 257},
411 {"32766", 10, 32766},
412 {"32767", 10, 32767},
413 {"32768", 10, 32768},
414 {"32769", 10, 32769},
415 {"65534", 10, 65534},
416 {"65535", 10, 65535},
417 {"65536", 10, 65536},
418 {"65537", 10, 65537},
419 {"2147483646", 10, 2147483646},
420 {"2147483647", 10, 2147483647},
421 {"2147483648", 10, 2147483648U},
422 {"2147483649", 10, 2147483649U},
423 {"4294967294", 10, 4294967294U},
424 {"4294967295", 10, 4294967295U},
425 };
426 TEST_OK(kstrtou32, u32, "%u", test_u32_ok);
427}
428
429static void __init test_kstrtou32_fail(void)
430{
431 static DEFINE_TEST_FAIL(test_u32_fail) = {
432 {"-2", 10},
433 {"-1", 10},
434 {"4294967296", 10},
435 {"4294967297", 10},
436 {"9223372036854775806", 10},
437 {"9223372036854775807", 10},
438 {"9223372036854775808", 10},
439 {"9223372036854775809", 10},
440 {"18446744073709551614", 10},
441 {"18446744073709551615", 10},
442 {"18446744073709551616", 10},
443 {"18446744073709551617", 10},
444 };
445 TEST_FAIL(kstrtou32, u32, "%u", test_u32_fail);
446}
447
448static void __init test_kstrtos32_ok(void)
449{
450 DECLARE_TEST_OK(s32, struct test_s32);
451 static DEFINE_TEST_OK(struct test_s32, test_s32_ok) = {
452 {"-128", 10, -128},
453 {"-127", 10, -127},
454 {"-1", 10, -1},
455 {"0", 10, 0},
456 {"1", 10, 1},
457 {"126", 10, 126},
458 {"127", 10, 127},
459 {"128", 10, 128},
460 {"129", 10, 129},
461 {"254", 10, 254},
462 {"255", 10, 255},
463 {"256", 10, 256},
464 {"257", 10, 257},
465 {"32766", 10, 32766},
466 {"32767", 10, 32767},
467 {"32768", 10, 32768},
468 {"32769", 10, 32769},
469 {"65534", 10, 65534},
470 {"65535", 10, 65535},
471 {"65536", 10, 65536},
472 {"65537", 10, 65537},
473 {"2147483646", 10, 2147483646},
474 {"2147483647", 10, 2147483647},
475 };
476 TEST_OK(kstrtos32, s32, "%d", test_s32_ok);
477}
478
479static void __init test_kstrtos32_fail(void)
480{
481 static DEFINE_TEST_FAIL(test_s32_fail) = {
482 {"2147483648", 10},
483 {"2147483649", 10},
484 {"4294967294", 10},
485 {"4294967295", 10},
486 {"4294967296", 10},
487 {"4294967297", 10},
488 {"9223372036854775806", 10},
489 {"9223372036854775807", 10},
490 {"9223372036854775808", 10},
491 {"9223372036854775809", 10},
492 {"18446744073709551614", 10},
493 {"18446744073709551615", 10},
494 {"18446744073709551616", 10},
495 {"18446744073709551617", 10},
496 };
497 TEST_FAIL(kstrtos32, s32, "%d", test_s32_fail);
498}
499
500static void __init test_kstrtou16_ok(void)
501{
502 DECLARE_TEST_OK(u16, struct test_u16);
503 static DEFINE_TEST_OK(struct test_u16, test_u16_ok) = {
504 {"0", 10, 0},
505 {"1", 10, 1},
506 {"126", 10, 126},
507 {"127", 10, 127},
508 {"128", 10, 128},
509 {"129", 10, 129},
510 {"254", 10, 254},
511 {"255", 10, 255},
512 {"256", 10, 256},
513 {"257", 10, 257},
514 {"32766", 10, 32766},
515 {"32767", 10, 32767},
516 {"32768", 10, 32768},
517 {"32769", 10, 32769},
518 {"65534", 10, 65534},
519 {"65535", 10, 65535},
520 };
521 TEST_OK(kstrtou16, u16, "%hu", test_u16_ok);
522}
523
524static void __init test_kstrtou16_fail(void)
525{
526 static DEFINE_TEST_FAIL(test_u16_fail) = {
527 {"-2", 10},
528 {"-1", 10},
529 {"65536", 10},
530 {"65537", 10},
531 {"2147483646", 10},
532 {"2147483647", 10},
533 {"2147483648", 10},
534 {"2147483649", 10},
535 {"4294967294", 10},
536 {"4294967295", 10},
537 {"4294967296", 10},
538 {"4294967297", 10},
539 {"9223372036854775806", 10},
540 {"9223372036854775807", 10},
541 {"9223372036854775808", 10},
542 {"9223372036854775809", 10},
543 {"18446744073709551614", 10},
544 {"18446744073709551615", 10},
545 {"18446744073709551616", 10},
546 {"18446744073709551617", 10},
547 };
548 TEST_FAIL(kstrtou16, u16, "%hu", test_u16_fail);
549}
550
551static void __init test_kstrtos16_ok(void)
552{
553 DECLARE_TEST_OK(s16, struct test_s16);
554 static DEFINE_TEST_OK(struct test_s16, test_s16_ok) = {
555 {"-130", 10, -130},
556 {"-129", 10, -129},
557 {"-128", 10, -128},
558 {"-127", 10, -127},
559 {"-1", 10, -1},
560 {"0", 10, 0},
561 {"1", 10, 1},
562 {"126", 10, 126},
563 {"127", 10, 127},
564 {"128", 10, 128},
565 {"129", 10, 129},
566 {"254", 10, 254},
567 {"255", 10, 255},
568 {"256", 10, 256},
569 {"257", 10, 257},
570 {"32766", 10, 32766},
571 {"32767", 10, 32767},
572 };
573 TEST_OK(kstrtos16, s16, "%hd", test_s16_ok);
574}
575
576static void __init test_kstrtos16_fail(void)
577{
578 static DEFINE_TEST_FAIL(test_s16_fail) = {
579 {"32768", 10},
580 {"32769", 10},
581 {"65534", 10},
582 {"65535", 10},
583 {"65536", 10},
584 {"65537", 10},
585 {"2147483646", 10},
586 {"2147483647", 10},
587 {"2147483648", 10},
588 {"2147483649", 10},
589 {"4294967294", 10},
590 {"4294967295", 10},
591 {"4294967296", 10},
592 {"4294967297", 10},
593 {"9223372036854775806", 10},
594 {"9223372036854775807", 10},
595 {"9223372036854775808", 10},
596 {"9223372036854775809", 10},
597 {"18446744073709551614", 10},
598 {"18446744073709551615", 10},
599 {"18446744073709551616", 10},
600 {"18446744073709551617", 10},
601 };
602 TEST_FAIL(kstrtos16, s16, "%hd", test_s16_fail);
603}
604
605static void __init test_kstrtou8_ok(void)
606{
607 DECLARE_TEST_OK(u8, struct test_u8);
608 static DEFINE_TEST_OK(struct test_u8, test_u8_ok) = {
609 {"0", 10, 0},
610 {"1", 10, 1},
611 {"126", 10, 126},
612 {"127", 10, 127},
613 {"128", 10, 128},
614 {"129", 10, 129},
615 {"254", 10, 254},
616 {"255", 10, 255},
617 };
618 TEST_OK(kstrtou8, u8, "%hhu", test_u8_ok);
619}
620
621static void __init test_kstrtou8_fail(void)
622{
623 static DEFINE_TEST_FAIL(test_u8_fail) = {
624 {"-2", 10},
625 {"-1", 10},
626 {"256", 10},
627 {"257", 10},
628 {"32766", 10},
629 {"32767", 10},
630 {"32768", 10},
631 {"32769", 10},
632 {"65534", 10},
633 {"65535", 10},
634 {"65536", 10},
635 {"65537", 10},
636 {"2147483646", 10},
637 {"2147483647", 10},
638 {"2147483648", 10},
639 {"2147483649", 10},
640 {"4294967294", 10},
641 {"4294967295", 10},
642 {"4294967296", 10},
643 {"4294967297", 10},
644 {"9223372036854775806", 10},
645 {"9223372036854775807", 10},
646 {"9223372036854775808", 10},
647 {"9223372036854775809", 10},
648 {"18446744073709551614", 10},
649 {"18446744073709551615", 10},
650 {"18446744073709551616", 10},
651 {"18446744073709551617", 10},
652 };
653 TEST_FAIL(kstrtou8, u8, "%hhu", test_u8_fail);
654}
655
656static void __init test_kstrtos8_ok(void)
657{
658 DECLARE_TEST_OK(s8, struct test_s8);
659 static DEFINE_TEST_OK(struct test_s8, test_s8_ok) = {
660 {"-128", 10, -128},
661 {"-127", 10, -127},
662 {"-1", 10, -1},
663 {"0", 10, 0},
664 {"1", 10, 1},
665 {"126", 10, 126},
666 {"127", 10, 127},
667 };
668 TEST_OK(kstrtos8, s8, "%hhd", test_s8_ok);
669}
670
671static void __init test_kstrtos8_fail(void)
672{
673 static DEFINE_TEST_FAIL(test_s8_fail) = {
674 {"-130", 10},
675 {"-129", 10},
676 {"128", 10},
677 {"129", 10},
678 {"254", 10},
679 {"255", 10},
680 {"256", 10},
681 {"257", 10},
682 {"32766", 10},
683 {"32767", 10},
684 {"32768", 10},
685 {"32769", 10},
686 {"65534", 10},
687 {"65535", 10},
688 {"65536", 10},
689 {"65537", 10},
690 {"2147483646", 10},
691 {"2147483647", 10},
692 {"2147483648", 10},
693 {"2147483649", 10},
694 {"4294967294", 10},
695 {"4294967295", 10},
696 {"4294967296", 10},
697 {"4294967297", 10},
698 {"9223372036854775806", 10},
699 {"9223372036854775807", 10},
700 {"9223372036854775808", 10},
701 {"9223372036854775809", 10},
702 {"18446744073709551614", 10},
703 {"18446744073709551615", 10},
704 {"18446744073709551616", 10},
705 {"18446744073709551617", 10},
706 };
707 TEST_FAIL(kstrtos8, s8, "%hhd", test_s8_fail);
708}
709
710static int __init test_kstrtox_init(void)
711{
712 test_kstrtoull_ok();
713 test_kstrtoull_fail();
714 test_kstrtoll_ok();
715 test_kstrtoll_fail();
716
717 test_kstrtou64_ok();
718 test_kstrtou64_fail();
719 test_kstrtos64_ok();
720 test_kstrtos64_fail();
721
722 test_kstrtou32_ok();
723 test_kstrtou32_fail();
724 test_kstrtos32_ok();
725 test_kstrtos32_fail();
726
727 test_kstrtou16_ok();
728 test_kstrtou16_fail();
729 test_kstrtos16_ok();
730 test_kstrtos16_fail();
731
732 test_kstrtou8_ok();
733 test_kstrtou8_fail();
734 test_kstrtos8_ok();
735 test_kstrtos8_fail();
736 return -EINVAL;
737}
738module_init(test_kstrtox_init);
739MODULE_LICENSE("Dual BSD/GPL");
diff --git a/lib/textsearch.c b/lib/textsearch.c
index d608331b3e47..e0cc0146ae62 100644
--- a/lib/textsearch.c
+++ b/lib/textsearch.c
@@ -13,7 +13,7 @@
13 * 13 *
14 * INTRODUCTION 14 * INTRODUCTION
15 * 15 *
16 * The textsearch infrastructure provides text searching facitilies for 16 * The textsearch infrastructure provides text searching facilities for
17 * both linear and non-linear data. Individual search algorithms are 17 * both linear and non-linear data. Individual search algorithms are
18 * implemented in modules and chosen by the user. 18 * implemented in modules and chosen by the user.
19 * 19 *
@@ -43,7 +43,7 @@
43 * to the algorithm to store persistent variables. 43 * to the algorithm to store persistent variables.
44 * (4) Core eventually resets the search offset and forwards the find() 44 * (4) Core eventually resets the search offset and forwards the find()
45 * request to the algorithm. 45 * request to the algorithm.
46 * (5) Algorithm calls get_next_block() provided by the user continously 46 * (5) Algorithm calls get_next_block() provided by the user continuously
47 * to fetch the data to be searched in block by block. 47 * to fetch the data to be searched in block by block.
48 * (6) Algorithm invokes finish() after the last call to get_next_block 48 * (6) Algorithm invokes finish() after the last call to get_next_block
49 * to clean up any leftovers from get_next_block. (Optional) 49 * to clean up any leftovers from get_next_block. (Optional)
@@ -58,15 +58,15 @@
58 * the pattern to look for and flags. As a flag, you can set TS_IGNORECASE 58 * the pattern to look for and flags. As a flag, you can set TS_IGNORECASE
59 * to perform case insensitive matching. But it might slow down 59 * to perform case insensitive matching. But it might slow down
60 * performance of algorithm, so you should use it at own your risk. 60 * performance of algorithm, so you should use it at own your risk.
61 * The returned configuration may then be used for an arbitary 61 * The returned configuration may then be used for an arbitrary
62 * amount of times and even in parallel as long as a separate struct 62 * amount of times and even in parallel as long as a separate struct
63 * ts_state variable is provided to every instance. 63 * ts_state variable is provided to every instance.
64 * 64 *
65 * The actual search is performed by either calling textsearch_find_- 65 * The actual search is performed by either calling textsearch_find_-
66 * continuous() for linear data or by providing an own get_next_block() 66 * continuous() for linear data or by providing an own get_next_block()
67 * implementation and calling textsearch_find(). Both functions return 67 * implementation and calling textsearch_find(). Both functions return
68 * the position of the first occurrence of the patern or UINT_MAX if 68 * the position of the first occurrence of the pattern or UINT_MAX if
69 * no match was found. Subsequent occurences can be found by calling 69 * no match was found. Subsequent occurrences can be found by calling
70 * textsearch_next() regardless of the linearity of the data. 70 * textsearch_next() regardless of the linearity of the data.
71 * 71 *
72 * Once you're done using a configuration it must be given back via 72 * Once you're done using a configuration it must be given back via
diff --git a/lib/timerqueue.c b/lib/timerqueue.c
new file mode 100644
index 000000000000..191176a43e9a
--- /dev/null
+++ b/lib/timerqueue.c
@@ -0,0 +1,107 @@
1/*
2 * Generic Timer-queue
3 *
4 * Manages a simple queue of timers, ordered by expiration time.
5 * Uses rbtrees for quick list adds and expiration.
6 *
7 * NOTE: All of the following functions need to be serialized
8 * to avoid races. No locking is done by this library code.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
24
25#include <linux/timerqueue.h>
26#include <linux/rbtree.h>
27#include <linux/module.h>
28
29/**
30 * timerqueue_add - Adds timer to timerqueue.
31 *
32 * @head: head of timerqueue
33 * @node: timer node to be added
34 *
35 * Adds the timer node to the timerqueue, sorted by the
36 * node's expires value.
37 */
38void timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node)
39{
40 struct rb_node **p = &head->head.rb_node;
41 struct rb_node *parent = NULL;
42 struct timerqueue_node *ptr;
43
44 /* Make sure we don't add nodes that are already added */
45 WARN_ON_ONCE(!RB_EMPTY_NODE(&node->node));
46
47 while (*p) {
48 parent = *p;
49 ptr = rb_entry(parent, struct timerqueue_node, node);
50 if (node->expires.tv64 < ptr->expires.tv64)
51 p = &(*p)->rb_left;
52 else
53 p = &(*p)->rb_right;
54 }
55 rb_link_node(&node->node, parent, p);
56 rb_insert_color(&node->node, &head->head);
57
58 if (!head->next || node->expires.tv64 < head->next->expires.tv64)
59 head->next = node;
60}
61EXPORT_SYMBOL_GPL(timerqueue_add);
62
63/**
64 * timerqueue_del - Removes a timer from the timerqueue.
65 *
66 * @head: head of timerqueue
67 * @node: timer node to be removed
68 *
69 * Removes the timer node from the timerqueue.
70 */
71void timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node)
72{
73 WARN_ON_ONCE(RB_EMPTY_NODE(&node->node));
74
75 /* update next pointer */
76 if (head->next == node) {
77 struct rb_node *rbn = rb_next(&node->node);
78
79 head->next = rbn ?
80 rb_entry(rbn, struct timerqueue_node, node) : NULL;
81 }
82 rb_erase(&node->node, &head->head);
83 RB_CLEAR_NODE(&node->node);
84}
85EXPORT_SYMBOL_GPL(timerqueue_del);
86
87/**
88 * timerqueue_iterate_next - Returns the timer after the provided timer
89 *
90 * @node: Pointer to a timer.
91 *
92 * Provides the timer that is after the given node. This is used, when
93 * necessary, to iterate through the list of timers in a timer list
94 * without modifying the list.
95 */
96struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node)
97{
98 struct rb_node *next;
99
100 if (!node)
101 return NULL;
102 next = rb_next(&node->node);
103 if (!next)
104 return NULL;
105 return container_of(next, struct timerqueue_node, node);
106}
107EXPORT_SYMBOL_GPL(timerqueue_iterate_next);
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 7af9d841c43b..4365df31a1d5 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -120,147 +120,6 @@ long long simple_strtoll(const char *cp, char **endp, unsigned int base)
120} 120}
121EXPORT_SYMBOL(simple_strtoll); 121EXPORT_SYMBOL(simple_strtoll);
122 122
123/**
124 * strict_strtoul - convert a string to an unsigned long strictly
125 * @cp: The string to be converted
126 * @base: The number base to use
127 * @res: The converted result value
128 *
129 * strict_strtoul converts a string to an unsigned long only if the
130 * string is really an unsigned long string, any string containing
131 * any invalid char at the tail will be rejected and -EINVAL is returned,
132 * only a newline char at the tail is acceptible because people generally
133 * change a module parameter in the following way:
134 *
135 * echo 1024 > /sys/module/e1000/parameters/copybreak
136 *
137 * echo will append a newline to the tail.
138 *
139 * It returns 0 if conversion is successful and *res is set to the converted
140 * value, otherwise it returns -EINVAL and *res is set to 0.
141 *
142 * simple_strtoul just ignores the successive invalid characters and
143 * return the converted value of prefix part of the string.
144 */
145int strict_strtoul(const char *cp, unsigned int base, unsigned long *res)
146{
147 char *tail;
148 unsigned long val;
149
150 *res = 0;
151 if (!*cp)
152 return -EINVAL;
153
154 val = simple_strtoul(cp, &tail, base);
155 if (tail == cp)
156 return -EINVAL;
157
158 if ((tail[0] == '\0') || (tail[0] == '\n' && tail[1] == '\0')) {
159 *res = val;
160 return 0;
161 }
162
163 return -EINVAL;
164}
165EXPORT_SYMBOL(strict_strtoul);
166
167/**
168 * strict_strtol - convert a string to a long strictly
169 * @cp: The string to be converted
170 * @base: The number base to use
171 * @res: The converted result value
172 *
173 * strict_strtol is similiar to strict_strtoul, but it allows the first
174 * character of a string is '-'.
175 *
176 * It returns 0 if conversion is successful and *res is set to the converted
177 * value, otherwise it returns -EINVAL and *res is set to 0.
178 */
179int strict_strtol(const char *cp, unsigned int base, long *res)
180{
181 int ret;
182 if (*cp == '-') {
183 ret = strict_strtoul(cp + 1, base, (unsigned long *)res);
184 if (!ret)
185 *res = -(*res);
186 } else {
187 ret = strict_strtoul(cp, base, (unsigned long *)res);
188 }
189
190 return ret;
191}
192EXPORT_SYMBOL(strict_strtol);
193
194/**
195 * strict_strtoull - convert a string to an unsigned long long strictly
196 * @cp: The string to be converted
197 * @base: The number base to use
198 * @res: The converted result value
199 *
200 * strict_strtoull converts a string to an unsigned long long only if the
201 * string is really an unsigned long long string, any string containing
202 * any invalid char at the tail will be rejected and -EINVAL is returned,
203 * only a newline char at the tail is acceptible because people generally
204 * change a module parameter in the following way:
205 *
206 * echo 1024 > /sys/module/e1000/parameters/copybreak
207 *
208 * echo will append a newline to the tail of the string.
209 *
210 * It returns 0 if conversion is successful and *res is set to the converted
211 * value, otherwise it returns -EINVAL and *res is set to 0.
212 *
213 * simple_strtoull just ignores the successive invalid characters and
214 * return the converted value of prefix part of the string.
215 */
216int strict_strtoull(const char *cp, unsigned int base, unsigned long long *res)
217{
218 char *tail;
219 unsigned long long val;
220
221 *res = 0;
222 if (!*cp)
223 return -EINVAL;
224
225 val = simple_strtoull(cp, &tail, base);
226 if (tail == cp)
227 return -EINVAL;
228 if ((tail[0] == '\0') || (tail[0] == '\n' && tail[1] == '\0')) {
229 *res = val;
230 return 0;
231 }
232
233 return -EINVAL;
234}
235EXPORT_SYMBOL(strict_strtoull);
236
237/**
238 * strict_strtoll - convert a string to a long long strictly
239 * @cp: The string to be converted
240 * @base: The number base to use
241 * @res: The converted result value
242 *
243 * strict_strtoll is similiar to strict_strtoull, but it allows the first
244 * character of a string is '-'.
245 *
246 * It returns 0 if conversion is successful and *res is set to the converted
247 * value, otherwise it returns -EINVAL and *res is set to 0.
248 */
249int strict_strtoll(const char *cp, unsigned int base, long long *res)
250{
251 int ret;
252 if (*cp == '-') {
253 ret = strict_strtoull(cp + 1, base, (unsigned long long *)res);
254 if (!ret)
255 *res = -(*res);
256 } else {
257 ret = strict_strtoull(cp, base, (unsigned long long *)res);
258 }
259
260 return ret;
261}
262EXPORT_SYMBOL(strict_strtoll);
263
264static noinline_for_stack 123static noinline_for_stack
265int skip_atoi(const char **s) 124int skip_atoi(const char **s)
266{ 125{
@@ -574,7 +433,9 @@ char *symbol_string(char *buf, char *end, void *ptr,
574 unsigned long value = (unsigned long) ptr; 433 unsigned long value = (unsigned long) ptr;
575#ifdef CONFIG_KALLSYMS 434#ifdef CONFIG_KALLSYMS
576 char sym[KSYM_SYMBOL_LEN]; 435 char sym[KSYM_SYMBOL_LEN];
577 if (ext != 'f' && ext != 's') 436 if (ext == 'B')
437 sprint_backtrace(sym, value);
438 else if (ext != 'f' && ext != 's')
578 sprint_symbol(sym, value); 439 sprint_symbol(sym, value);
579 else 440 else
580 kallsyms_lookup(value, NULL, NULL, NULL, sym); 441 kallsyms_lookup(value, NULL, NULL, NULL, sym);
@@ -805,6 +666,8 @@ char *ip6_compressed_string(char *p, const char *addr)
805 colonpos = i; 666 colonpos = i;
806 } 667 }
807 } 668 }
669 if (longest == 1) /* don't compress a single 0 */
670 colonpos = -1;
808 671
809 /* emit address */ 672 /* emit address */
810 for (i = 0; i < range; i++) { 673 for (i = 0; i < range; i++) {
@@ -936,6 +799,8 @@ char *uuid_string(char *buf, char *end, const u8 *addr,
936 return string(buf, end, uuid, spec); 799 return string(buf, end, uuid, spec);
937} 800}
938 801
802int kptr_restrict __read_mostly;
803
939/* 804/*
940 * Show a '%p' thing. A kernel extension is that the '%p' is followed 805 * Show a '%p' thing. A kernel extension is that the '%p' is followed
941 * by an extra set of alphanumeric characters that are extended format 806 * by an extra set of alphanumeric characters that are extended format
@@ -947,6 +812,7 @@ char *uuid_string(char *buf, char *end, const u8 *addr,
947 * - 'f' For simple symbolic function names without offset 812 * - 'f' For simple symbolic function names without offset
948 * - 'S' For symbolic direct pointers with offset 813 * - 'S' For symbolic direct pointers with offset
949 * - 's' For symbolic direct pointers without offset 814 * - 's' For symbolic direct pointers without offset
815 * - 'B' For backtraced symbolic direct pointers with offset
950 * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref] 816 * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref]
951 * - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201] 817 * - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201]
952 * - 'M' For a 6-byte MAC address, it prints the address in the 818 * - 'M' For a 6-byte MAC address, it prints the address in the
@@ -962,7 +828,7 @@ char *uuid_string(char *buf, char *end, const u8 *addr,
962 * IPv4 uses dot-separated decimal with leading 0's (010.123.045.006) 828 * IPv4 uses dot-separated decimal with leading 0's (010.123.045.006)
963 * - '[Ii]4[hnbl]' IPv4 addresses in host, network, big or little endian order 829 * - '[Ii]4[hnbl]' IPv4 addresses in host, network, big or little endian order
964 * - 'I6c' for IPv6 addresses printed as specified by 830 * - 'I6c' for IPv6 addresses printed as specified by
965 * http://tools.ietf.org/html/draft-ietf-6man-text-addr-representation-00 831 * http://tools.ietf.org/html/rfc5952
966 * - 'U' For a 16 byte UUID/GUID, it prints the UUID/GUID in the form 832 * - 'U' For a 16 byte UUID/GUID, it prints the UUID/GUID in the form
967 * "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" 833 * "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
968 * Options for %pU are: 834 * Options for %pU are:
@@ -979,6 +845,7 @@ char *uuid_string(char *buf, char *end, const u8 *addr,
979 * Implements a "recursive vsnprintf". 845 * Implements a "recursive vsnprintf".
980 * Do not use this feature without some mechanism to verify the 846 * Do not use this feature without some mechanism to verify the
981 * correctness of the format string and va_list arguments. 847 * correctness of the format string and va_list arguments.
848 * - 'K' For a kernel pointer that should be hidden from unprivileged users
982 * 849 *
983 * Note: The difference between 'S' and 'F' is that on ia64 and ppc64 850 * Note: The difference between 'S' and 'F' is that on ia64 and ppc64
984 * function pointers are really function descriptors, which contain a 851 * function pointers are really function descriptors, which contain a
@@ -988,8 +855,15 @@ static noinline_for_stack
988char *pointer(const char *fmt, char *buf, char *end, void *ptr, 855char *pointer(const char *fmt, char *buf, char *end, void *ptr,
989 struct printf_spec spec) 856 struct printf_spec spec)
990{ 857{
991 if (!ptr) 858 if (!ptr && *fmt != 'K') {
859 /*
860 * Print (null) with the same width as a pointer so it makes
861 * tabular output look nice.
862 */
863 if (spec.field_width == -1)
864 spec.field_width = 2 * sizeof(void *);
992 return string(buf, end, "(null)", spec); 865 return string(buf, end, "(null)", spec);
866 }
993 867
994 switch (*fmt) { 868 switch (*fmt) {
995 case 'F': 869 case 'F':
@@ -998,6 +872,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
998 /* Fallthrough */ 872 /* Fallthrough */
999 case 'S': 873 case 'S':
1000 case 's': 874 case 's':
875 case 'B':
1001 return symbol_string(buf, end, ptr, spec, *fmt); 876 return symbol_string(buf, end, ptr, spec, *fmt);
1002 case 'R': 877 case 'R':
1003 case 'r': 878 case 'r':
@@ -1025,13 +900,28 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
1025 case 'U': 900 case 'U':
1026 return uuid_string(buf, end, ptr, spec, fmt); 901 return uuid_string(buf, end, ptr, spec, fmt);
1027 case 'V': 902 case 'V':
1028 return buf + vsnprintf(buf, end - buf, 903 return buf + vsnprintf(buf, end > buf ? end - buf : 0,
1029 ((struct va_format *)ptr)->fmt, 904 ((struct va_format *)ptr)->fmt,
1030 *(((struct va_format *)ptr)->va)); 905 *(((struct va_format *)ptr)->va));
906 case 'K':
907 /*
908 * %pK cannot be used in IRQ context because its test
909 * for CAP_SYSLOG would be meaningless.
910 */
911 if (in_irq() || in_serving_softirq() || in_nmi()) {
912 if (spec.field_width == -1)
913 spec.field_width = 2 * sizeof(void *);
914 return string(buf, end, "pK-error", spec);
915 }
916 if (!((kptr_restrict == 0) ||
917 (kptr_restrict == 1 &&
918 has_capability_noaudit(current, CAP_SYSLOG))))
919 ptr = NULL;
920 break;
1031 } 921 }
1032 spec.flags |= SMALL; 922 spec.flags |= SMALL;
1033 if (spec.field_width == -1) { 923 if (spec.field_width == -1) {
1034 spec.field_width = 2*sizeof(void *); 924 spec.field_width = 2 * sizeof(void *);
1035 spec.flags |= ZEROPAD; 925 spec.flags |= ZEROPAD;
1036 } 926 }
1037 spec.base = 16; 927 spec.base = 16;
@@ -1250,6 +1140,7 @@ qualifier:
1250 * %ps output the name of a text symbol without offset 1140 * %ps output the name of a text symbol without offset
1251 * %pF output the name of a function pointer with its offset 1141 * %pF output the name of a function pointer with its offset
1252 * %pf output the name of a function pointer without its offset 1142 * %pf output the name of a function pointer without its offset
1143 * %pB output the name of a backtrace symbol with its offset
1253 * %pR output the address range in a struct resource with decoded flags 1144 * %pR output the address range in a struct resource with decoded flags
1254 * %pr output the address range in a struct resource with raw flags 1145 * %pr output the address range in a struct resource with raw flags
1255 * %pM output a 6-byte MAC address with colons 1146 * %pM output a 6-byte MAC address with colons
@@ -1272,8 +1163,7 @@ qualifier:
1272 * return is greater than or equal to @size, the resulting 1163 * return is greater than or equal to @size, the resulting
1273 * string is truncated. 1164 * string is truncated.
1274 * 1165 *
1275 * Call this function if you are already dealing with a va_list. 1166 * If you're not already dealing with a va_list consider using snprintf().
1276 * You probably want snprintf() instead.
1277 */ 1167 */
1278int vsnprintf(char *buf, size_t size, const char *fmt, va_list args) 1168int vsnprintf(char *buf, size_t size, const char *fmt, va_list args)
1279{ 1169{
@@ -1444,11 +1334,10 @@ EXPORT_SYMBOL(vsnprintf);
1444 * @args: Arguments for the format string 1334 * @args: Arguments for the format string
1445 * 1335 *
1446 * The return value is the number of characters which have been written into 1336 * The return value is the number of characters which have been written into
1447 * the @buf not including the trailing '\0'. If @size is <= 0 the function 1337 * the @buf not including the trailing '\0'. If @size is == 0 the function
1448 * returns 0. 1338 * returns 0.
1449 * 1339 *
1450 * Call this function if you are already dealing with a va_list. 1340 * If you're not already dealing with a va_list consider using scnprintf().
1451 * You probably want scnprintf() instead.
1452 * 1341 *
1453 * See the vsnprintf() documentation for format string extensions over C99. 1342 * See the vsnprintf() documentation for format string extensions over C99.
1454 */ 1343 */
@@ -1458,7 +1347,11 @@ int vscnprintf(char *buf, size_t size, const char *fmt, va_list args)
1458 1347
1459 i = vsnprintf(buf, size, fmt, args); 1348 i = vsnprintf(buf, size, fmt, args);
1460 1349
1461 return (i >= size) ? (size - 1) : i; 1350 if (likely(i < size))
1351 return i;
1352 if (size != 0)
1353 return size - 1;
1354 return 0;
1462} 1355}
1463EXPORT_SYMBOL(vscnprintf); 1356EXPORT_SYMBOL(vscnprintf);
1464 1357
@@ -1497,7 +1390,7 @@ EXPORT_SYMBOL(snprintf);
1497 * @...: Arguments for the format string 1390 * @...: Arguments for the format string
1498 * 1391 *
1499 * The return value is the number of characters written into @buf not including 1392 * The return value is the number of characters written into @buf not including
1500 * the trailing '\0'. If @size is <= 0 the function returns 0. 1393 * the trailing '\0'. If @size is == 0 the function returns 0.
1501 */ 1394 */
1502 1395
1503int scnprintf(char *buf, size_t size, const char *fmt, ...) 1396int scnprintf(char *buf, size_t size, const char *fmt, ...)
@@ -1506,10 +1399,10 @@ int scnprintf(char *buf, size_t size, const char *fmt, ...)
1506 int i; 1399 int i;
1507 1400
1508 va_start(args, fmt); 1401 va_start(args, fmt);
1509 i = vsnprintf(buf, size, fmt, args); 1402 i = vscnprintf(buf, size, fmt, args);
1510 va_end(args); 1403 va_end(args);
1511 1404
1512 return (i >= size) ? (size - 1) : i; 1405 return i;
1513} 1406}
1514EXPORT_SYMBOL(scnprintf); 1407EXPORT_SYMBOL(scnprintf);
1515 1408
@@ -1523,8 +1416,7 @@ EXPORT_SYMBOL(scnprintf);
1523 * into @buf. Use vsnprintf() or vscnprintf() in order to avoid 1416 * into @buf. Use vsnprintf() or vscnprintf() in order to avoid
1524 * buffer overflows. 1417 * buffer overflows.
1525 * 1418 *
1526 * Call this function if you are already dealing with a va_list. 1419 * If you're not already dealing with a va_list consider using sprintf().
1527 * You probably want sprintf() instead.
1528 * 1420 *
1529 * See the vsnprintf() documentation for format string extensions over C99. 1421 * See the vsnprintf() documentation for format string extensions over C99.
1530 */ 1422 */
diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig
new file mode 100644
index 000000000000..60a6088d0e5e
--- /dev/null
+++ b/lib/xz/Kconfig
@@ -0,0 +1,59 @@
1config XZ_DEC
2 tristate "XZ decompression support"
3 select CRC32
4 help
5 LZMA2 compression algorithm and BCJ filters are supported using
6 the .xz file format as the container. For integrity checking,
7 CRC32 is supported. See Documentation/xz.txt for more information.
8
9config XZ_DEC_X86
10 bool "x86 BCJ filter decoder" if EXPERT
11 default y
12 depends on XZ_DEC
13 select XZ_DEC_BCJ
14
15config XZ_DEC_POWERPC
16 bool "PowerPC BCJ filter decoder" if EXPERT
17 default y
18 depends on XZ_DEC
19 select XZ_DEC_BCJ
20
21config XZ_DEC_IA64
22 bool "IA-64 BCJ filter decoder" if EXPERT
23 default y
24 depends on XZ_DEC
25 select XZ_DEC_BCJ
26
27config XZ_DEC_ARM
28 bool "ARM BCJ filter decoder" if EXPERT
29 default y
30 depends on XZ_DEC
31 select XZ_DEC_BCJ
32
33config XZ_DEC_ARMTHUMB
34 bool "ARM-Thumb BCJ filter decoder" if EXPERT
35 default y
36 depends on XZ_DEC
37 select XZ_DEC_BCJ
38
39config XZ_DEC_SPARC
40 bool "SPARC BCJ filter decoder" if EXPERT
41 default y
42 depends on XZ_DEC
43 select XZ_DEC_BCJ
44
45config XZ_DEC_BCJ
46 bool
47 default n
48
49config XZ_DEC_TEST
50 tristate "XZ decompressor tester"
51 default n
52 depends on XZ_DEC
53 help
54 This allows passing .xz files to the in-kernel XZ decoder via
55 a character special file. It calculates CRC32 of the decompressed
56 data and writes diagnostics to the system log.
57
58 Unless you are developing the XZ decoder, you don't need this
59 and should say N.
diff --git a/lib/xz/Makefile b/lib/xz/Makefile
new file mode 100644
index 000000000000..a7fa7693f0f3
--- /dev/null
+++ b/lib/xz/Makefile
@@ -0,0 +1,5 @@
1obj-$(CONFIG_XZ_DEC) += xz_dec.o
2xz_dec-y := xz_dec_syms.o xz_dec_stream.o xz_dec_lzma2.o
3xz_dec-$(CONFIG_XZ_DEC_BCJ) += xz_dec_bcj.o
4
5obj-$(CONFIG_XZ_DEC_TEST) += xz_dec_test.o
diff --git a/lib/xz/xz_crc32.c b/lib/xz/xz_crc32.c
new file mode 100644
index 000000000000..34532d14fd4c
--- /dev/null
+++ b/lib/xz/xz_crc32.c
@@ -0,0 +1,59 @@
1/*
2 * CRC32 using the polynomial from IEEE-802.3
3 *
4 * Authors: Lasse Collin <lasse.collin@tukaani.org>
5 * Igor Pavlov <http://7-zip.org/>
6 *
7 * This file has been put into the public domain.
8 * You can do whatever you want with this file.
9 */
10
11/*
12 * This is not the fastest implementation, but it is pretty compact.
13 * The fastest versions of xz_crc32() on modern CPUs without hardware
14 * accelerated CRC instruction are 3-5 times as fast as this version,
15 * but they are bigger and use more memory for the lookup table.
16 */
17
18#include "xz_private.h"
19
20/*
21 * STATIC_RW_DATA is used in the pre-boot environment on some architectures.
22 * See <linux/decompress/mm.h> for details.
23 */
24#ifndef STATIC_RW_DATA
25# define STATIC_RW_DATA static
26#endif
27
28STATIC_RW_DATA uint32_t xz_crc32_table[256];
29
30XZ_EXTERN void xz_crc32_init(void)
31{
32 const uint32_t poly = 0xEDB88320;
33
34 uint32_t i;
35 uint32_t j;
36 uint32_t r;
37
38 for (i = 0; i < 256; ++i) {
39 r = i;
40 for (j = 0; j < 8; ++j)
41 r = (r >> 1) ^ (poly & ~((r & 1) - 1));
42
43 xz_crc32_table[i] = r;
44 }
45
46 return;
47}
48
49XZ_EXTERN uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc)
50{
51 crc = ~crc;
52
53 while (size != 0) {
54 crc = xz_crc32_table[*buf++ ^ (crc & 0xFF)] ^ (crc >> 8);
55 --size;
56 }
57
58 return ~crc;
59}
diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c
new file mode 100644
index 000000000000..e51e2558ca9d
--- /dev/null
+++ b/lib/xz/xz_dec_bcj.c
@@ -0,0 +1,561 @@
1/*
2 * Branch/Call/Jump (BCJ) filter decoders
3 *
4 * Authors: Lasse Collin <lasse.collin@tukaani.org>
5 * Igor Pavlov <http://7-zip.org/>
6 *
7 * This file has been put into the public domain.
8 * You can do whatever you want with this file.
9 */
10
11#include "xz_private.h"
12
13/*
14 * The rest of the file is inside this ifdef. It makes things a little more
15 * convenient when building without support for any BCJ filters.
16 */
17#ifdef XZ_DEC_BCJ
18
19struct xz_dec_bcj {
20 /* Type of the BCJ filter being used */
21 enum {
22 BCJ_X86 = 4, /* x86 or x86-64 */
23 BCJ_POWERPC = 5, /* Big endian only */
24 BCJ_IA64 = 6, /* Big or little endian */
25 BCJ_ARM = 7, /* Little endian only */
26 BCJ_ARMTHUMB = 8, /* Little endian only */
27 BCJ_SPARC = 9 /* Big or little endian */
28 } type;
29
30 /*
31 * Return value of the next filter in the chain. We need to preserve
32 * this information across calls, because we must not call the next
33 * filter anymore once it has returned XZ_STREAM_END.
34 */
35 enum xz_ret ret;
36
37 /* True if we are operating in single-call mode. */
38 bool single_call;
39
40 /*
41 * Absolute position relative to the beginning of the uncompressed
42 * data (in a single .xz Block). We care only about the lowest 32
43 * bits so this doesn't need to be uint64_t even with big files.
44 */
45 uint32_t pos;
46
47 /* x86 filter state */
48 uint32_t x86_prev_mask;
49
50 /* Temporary space to hold the variables from struct xz_buf */
51 uint8_t *out;
52 size_t out_pos;
53 size_t out_size;
54
55 struct {
56 /* Amount of already filtered data in the beginning of buf */
57 size_t filtered;
58
59 /* Total amount of data currently stored in buf */
60 size_t size;
61
62 /*
63 * Buffer to hold a mix of filtered and unfiltered data. This
64 * needs to be big enough to hold Alignment + 2 * Look-ahead:
65 *
66 * Type Alignment Look-ahead
67 * x86 1 4
68 * PowerPC 4 0
69 * IA-64 16 0
70 * ARM 4 0
71 * ARM-Thumb 2 2
72 * SPARC 4 0
73 */
74 uint8_t buf[16];
75 } temp;
76};
77
78#ifdef XZ_DEC_X86
79/*
80 * This is used to test the most significant byte of a memory address
81 * in an x86 instruction.
82 */
83static inline int bcj_x86_test_msbyte(uint8_t b)
84{
85 return b == 0x00 || b == 0xFF;
86}
87
88static size_t bcj_x86(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
89{
90 static const bool mask_to_allowed_status[8]
91 = { true, true, true, false, true, false, false, false };
92
93 static const uint8_t mask_to_bit_num[8] = { 0, 1, 2, 2, 3, 3, 3, 3 };
94
95 size_t i;
96 size_t prev_pos = (size_t)-1;
97 uint32_t prev_mask = s->x86_prev_mask;
98 uint32_t src;
99 uint32_t dest;
100 uint32_t j;
101 uint8_t b;
102
103 if (size <= 4)
104 return 0;
105
106 size -= 4;
107 for (i = 0; i < size; ++i) {
108 if ((buf[i] & 0xFE) != 0xE8)
109 continue;
110
111 prev_pos = i - prev_pos;
112 if (prev_pos > 3) {
113 prev_mask = 0;
114 } else {
115 prev_mask = (prev_mask << (prev_pos - 1)) & 7;
116 if (prev_mask != 0) {
117 b = buf[i + 4 - mask_to_bit_num[prev_mask]];
118 if (!mask_to_allowed_status[prev_mask]
119 || bcj_x86_test_msbyte(b)) {
120 prev_pos = i;
121 prev_mask = (prev_mask << 1) | 1;
122 continue;
123 }
124 }
125 }
126
127 prev_pos = i;
128
129 if (bcj_x86_test_msbyte(buf[i + 4])) {
130 src = get_unaligned_le32(buf + i + 1);
131 while (true) {
132 dest = src - (s->pos + (uint32_t)i + 5);
133 if (prev_mask == 0)
134 break;
135
136 j = mask_to_bit_num[prev_mask] * 8;
137 b = (uint8_t)(dest >> (24 - j));
138 if (!bcj_x86_test_msbyte(b))
139 break;
140
141 src = dest ^ (((uint32_t)1 << (32 - j)) - 1);
142 }
143
144 dest &= 0x01FFFFFF;
145 dest |= (uint32_t)0 - (dest & 0x01000000);
146 put_unaligned_le32(dest, buf + i + 1);
147 i += 4;
148 } else {
149 prev_mask = (prev_mask << 1) | 1;
150 }
151 }
152
153 prev_pos = i - prev_pos;
154 s->x86_prev_mask = prev_pos > 3 ? 0 : prev_mask << (prev_pos - 1);
155 return i;
156}
157#endif
158
159#ifdef XZ_DEC_POWERPC
160static size_t bcj_powerpc(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
161{
162 size_t i;
163 uint32_t instr;
164
165 for (i = 0; i + 4 <= size; i += 4) {
166 instr = get_unaligned_be32(buf + i);
167 if ((instr & 0xFC000003) == 0x48000001) {
168 instr &= 0x03FFFFFC;
169 instr -= s->pos + (uint32_t)i;
170 instr &= 0x03FFFFFC;
171 instr |= 0x48000001;
172 put_unaligned_be32(instr, buf + i);
173 }
174 }
175
176 return i;
177}
178#endif
179
180#ifdef XZ_DEC_IA64
181static size_t bcj_ia64(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
182{
183 static const uint8_t branch_table[32] = {
184 0, 0, 0, 0, 0, 0, 0, 0,
185 0, 0, 0, 0, 0, 0, 0, 0,
186 4, 4, 6, 6, 0, 0, 7, 7,
187 4, 4, 0, 0, 4, 4, 0, 0
188 };
189
190 /*
191 * The local variables take a little bit stack space, but it's less
192 * than what LZMA2 decoder takes, so it doesn't make sense to reduce
193 * stack usage here without doing that for the LZMA2 decoder too.
194 */
195
196 /* Loop counters */
197 size_t i;
198 size_t j;
199
200 /* Instruction slot (0, 1, or 2) in the 128-bit instruction word */
201 uint32_t slot;
202
203 /* Bitwise offset of the instruction indicated by slot */
204 uint32_t bit_pos;
205
206 /* bit_pos split into byte and bit parts */
207 uint32_t byte_pos;
208 uint32_t bit_res;
209
210 /* Address part of an instruction */
211 uint32_t addr;
212
213 /* Mask used to detect which instructions to convert */
214 uint32_t mask;
215
216 /* 41-bit instruction stored somewhere in the lowest 48 bits */
217 uint64_t instr;
218
219 /* Instruction normalized with bit_res for easier manipulation */
220 uint64_t norm;
221
222 for (i = 0; i + 16 <= size; i += 16) {
223 mask = branch_table[buf[i] & 0x1F];
224 for (slot = 0, bit_pos = 5; slot < 3; ++slot, bit_pos += 41) {
225 if (((mask >> slot) & 1) == 0)
226 continue;
227
228 byte_pos = bit_pos >> 3;
229 bit_res = bit_pos & 7;
230 instr = 0;
231 for (j = 0; j < 6; ++j)
232 instr |= (uint64_t)(buf[i + j + byte_pos])
233 << (8 * j);
234
235 norm = instr >> bit_res;
236
237 if (((norm >> 37) & 0x0F) == 0x05
238 && ((norm >> 9) & 0x07) == 0) {
239 addr = (norm >> 13) & 0x0FFFFF;
240 addr |= ((uint32_t)(norm >> 36) & 1) << 20;
241 addr <<= 4;
242 addr -= s->pos + (uint32_t)i;
243 addr >>= 4;
244
245 norm &= ~((uint64_t)0x8FFFFF << 13);
246 norm |= (uint64_t)(addr & 0x0FFFFF) << 13;
247 norm |= (uint64_t)(addr & 0x100000)
248 << (36 - 20);
249
250 instr &= (1 << bit_res) - 1;
251 instr |= norm << bit_res;
252
253 for (j = 0; j < 6; j++)
254 buf[i + j + byte_pos]
255 = (uint8_t)(instr >> (8 * j));
256 }
257 }
258 }
259
260 return i;
261}
262#endif
263
264#ifdef XZ_DEC_ARM
265static size_t bcj_arm(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
266{
267 size_t i;
268 uint32_t addr;
269
270 for (i = 0; i + 4 <= size; i += 4) {
271 if (buf[i + 3] == 0xEB) {
272 addr = (uint32_t)buf[i] | ((uint32_t)buf[i + 1] << 8)
273 | ((uint32_t)buf[i + 2] << 16);
274 addr <<= 2;
275 addr -= s->pos + (uint32_t)i + 8;
276 addr >>= 2;
277 buf[i] = (uint8_t)addr;
278 buf[i + 1] = (uint8_t)(addr >> 8);
279 buf[i + 2] = (uint8_t)(addr >> 16);
280 }
281 }
282
283 return i;
284}
285#endif
286
287#ifdef XZ_DEC_ARMTHUMB
288static size_t bcj_armthumb(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
289{
290 size_t i;
291 uint32_t addr;
292
293 for (i = 0; i + 4 <= size; i += 2) {
294 if ((buf[i + 1] & 0xF8) == 0xF0
295 && (buf[i + 3] & 0xF8) == 0xF8) {
296 addr = (((uint32_t)buf[i + 1] & 0x07) << 19)
297 | ((uint32_t)buf[i] << 11)
298 | (((uint32_t)buf[i + 3] & 0x07) << 8)
299 | (uint32_t)buf[i + 2];
300 addr <<= 1;
301 addr -= s->pos + (uint32_t)i + 4;
302 addr >>= 1;
303 buf[i + 1] = (uint8_t)(0xF0 | ((addr >> 19) & 0x07));
304 buf[i] = (uint8_t)(addr >> 11);
305 buf[i + 3] = (uint8_t)(0xF8 | ((addr >> 8) & 0x07));
306 buf[i + 2] = (uint8_t)addr;
307 i += 2;
308 }
309 }
310
311 return i;
312}
313#endif
314
315#ifdef XZ_DEC_SPARC
316static size_t bcj_sparc(struct xz_dec_bcj *s, uint8_t *buf, size_t size)
317{
318 size_t i;
319 uint32_t instr;
320
321 for (i = 0; i + 4 <= size; i += 4) {
322 instr = get_unaligned_be32(buf + i);
323 if ((instr >> 22) == 0x100 || (instr >> 22) == 0x1FF) {
324 instr <<= 2;
325 instr -= s->pos + (uint32_t)i;
326 instr >>= 2;
327 instr = ((uint32_t)0x40000000 - (instr & 0x400000))
328 | 0x40000000 | (instr & 0x3FFFFF);
329 put_unaligned_be32(instr, buf + i);
330 }
331 }
332
333 return i;
334}
335#endif
336
337/*
338 * Apply the selected BCJ filter. Update *pos and s->pos to match the amount
339 * of data that got filtered.
340 *
341 * NOTE: This is implemented as a switch statement to avoid using function
342 * pointers, which could be problematic in the kernel boot code, which must
343 * avoid pointers to static data (at least on x86).
344 */
345static void bcj_apply(struct xz_dec_bcj *s,
346 uint8_t *buf, size_t *pos, size_t size)
347{
348 size_t filtered;
349
350 buf += *pos;
351 size -= *pos;
352
353 switch (s->type) {
354#ifdef XZ_DEC_X86
355 case BCJ_X86:
356 filtered = bcj_x86(s, buf, size);
357 break;
358#endif
359#ifdef XZ_DEC_POWERPC
360 case BCJ_POWERPC:
361 filtered = bcj_powerpc(s, buf, size);
362 break;
363#endif
364#ifdef XZ_DEC_IA64
365 case BCJ_IA64:
366 filtered = bcj_ia64(s, buf, size);
367 break;
368#endif
369#ifdef XZ_DEC_ARM
370 case BCJ_ARM:
371 filtered = bcj_arm(s, buf, size);
372 break;
373#endif
374#ifdef XZ_DEC_ARMTHUMB
375 case BCJ_ARMTHUMB:
376 filtered = bcj_armthumb(s, buf, size);
377 break;
378#endif
379#ifdef XZ_DEC_SPARC
380 case BCJ_SPARC:
381 filtered = bcj_sparc(s, buf, size);
382 break;
383#endif
384 default:
385 /* Never reached but silence compiler warnings. */
386 filtered = 0;
387 break;
388 }
389
390 *pos += filtered;
391 s->pos += filtered;
392}
393
394/*
395 * Flush pending filtered data from temp to the output buffer.
396 * Move the remaining mixture of possibly filtered and unfiltered
397 * data to the beginning of temp.
398 */
399static void bcj_flush(struct xz_dec_bcj *s, struct xz_buf *b)
400{
401 size_t copy_size;
402
403 copy_size = min_t(size_t, s->temp.filtered, b->out_size - b->out_pos);
404 memcpy(b->out + b->out_pos, s->temp.buf, copy_size);
405 b->out_pos += copy_size;
406
407 s->temp.filtered -= copy_size;
408 s->temp.size -= copy_size;
409 memmove(s->temp.buf, s->temp.buf + copy_size, s->temp.size);
410}
411
412/*
413 * The BCJ filter functions are primitive in sense that they process the
414 * data in chunks of 1-16 bytes. To hide this issue, this function does
415 * some buffering.
416 */
417XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s,
418 struct xz_dec_lzma2 *lzma2,
419 struct xz_buf *b)
420{
421 size_t out_start;
422
423 /*
424 * Flush pending already filtered data to the output buffer. Return
425 * immediatelly if we couldn't flush everything, or if the next
426 * filter in the chain had already returned XZ_STREAM_END.
427 */
428 if (s->temp.filtered > 0) {
429 bcj_flush(s, b);
430 if (s->temp.filtered > 0)
431 return XZ_OK;
432
433 if (s->ret == XZ_STREAM_END)
434 return XZ_STREAM_END;
435 }
436
437 /*
438 * If we have more output space than what is currently pending in
439 * temp, copy the unfiltered data from temp to the output buffer
440 * and try to fill the output buffer by decoding more data from the
441 * next filter in the chain. Apply the BCJ filter on the new data
442 * in the output buffer. If everything cannot be filtered, copy it
443 * to temp and rewind the output buffer position accordingly.
444 */
445 if (s->temp.size < b->out_size - b->out_pos) {
446 out_start = b->out_pos;
447 memcpy(b->out + b->out_pos, s->temp.buf, s->temp.size);
448 b->out_pos += s->temp.size;
449
450 s->ret = xz_dec_lzma2_run(lzma2, b);
451 if (s->ret != XZ_STREAM_END
452 && (s->ret != XZ_OK || s->single_call))
453 return s->ret;
454
455 bcj_apply(s, b->out, &out_start, b->out_pos);
456
457 /*
458 * As an exception, if the next filter returned XZ_STREAM_END,
459 * we can do that too, since the last few bytes that remain
460 * unfiltered are meant to remain unfiltered.
461 */
462 if (s->ret == XZ_STREAM_END)
463 return XZ_STREAM_END;
464
465 s->temp.size = b->out_pos - out_start;
466 b->out_pos -= s->temp.size;
467 memcpy(s->temp.buf, b->out + b->out_pos, s->temp.size);
468 }
469
470 /*
471 * If we have unfiltered data in temp, try to fill by decoding more
472 * data from the next filter. Apply the BCJ filter on temp. Then we
473 * hopefully can fill the actual output buffer by copying filtered
474 * data from temp. A mix of filtered and unfiltered data may be left
475 * in temp; it will be taken care on the next call to this function.
476 */
477 if (s->temp.size > 0) {
478 /* Make b->out{,_pos,_size} temporarily point to s->temp. */
479 s->out = b->out;
480 s->out_pos = b->out_pos;
481 s->out_size = b->out_size;
482 b->out = s->temp.buf;
483 b->out_pos = s->temp.size;
484 b->out_size = sizeof(s->temp.buf);
485
486 s->ret = xz_dec_lzma2_run(lzma2, b);
487
488 s->temp.size = b->out_pos;
489 b->out = s->out;
490 b->out_pos = s->out_pos;
491 b->out_size = s->out_size;
492
493 if (s->ret != XZ_OK && s->ret != XZ_STREAM_END)
494 return s->ret;
495
496 bcj_apply(s, s->temp.buf, &s->temp.filtered, s->temp.size);
497
498 /*
499 * If the next filter returned XZ_STREAM_END, we mark that
500 * everything is filtered, since the last unfiltered bytes
501 * of the stream are meant to be left as is.
502 */
503 if (s->ret == XZ_STREAM_END)
504 s->temp.filtered = s->temp.size;
505
506 bcj_flush(s, b);
507 if (s->temp.filtered > 0)
508 return XZ_OK;
509 }
510
511 return s->ret;
512}
513
514XZ_EXTERN struct xz_dec_bcj *xz_dec_bcj_create(bool single_call)
515{
516 struct xz_dec_bcj *s = kmalloc(sizeof(*s), GFP_KERNEL);
517 if (s != NULL)
518 s->single_call = single_call;
519
520 return s;
521}
522
523XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id)
524{
525 switch (id) {
526#ifdef XZ_DEC_X86
527 case BCJ_X86:
528#endif
529#ifdef XZ_DEC_POWERPC
530 case BCJ_POWERPC:
531#endif
532#ifdef XZ_DEC_IA64
533 case BCJ_IA64:
534#endif
535#ifdef XZ_DEC_ARM
536 case BCJ_ARM:
537#endif
538#ifdef XZ_DEC_ARMTHUMB
539 case BCJ_ARMTHUMB:
540#endif
541#ifdef XZ_DEC_SPARC
542 case BCJ_SPARC:
543#endif
544 break;
545
546 default:
547 /* Unsupported Filter ID */
548 return XZ_OPTIONS_ERROR;
549 }
550
551 s->type = id;
552 s->ret = XZ_OK;
553 s->pos = 0;
554 s->x86_prev_mask = 0;
555 s->temp.filtered = 0;
556 s->temp.size = 0;
557
558 return XZ_OK;
559}
560
561#endif
diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c
new file mode 100644
index 000000000000..a6cdc969ea42
--- /dev/null
+++ b/lib/xz/xz_dec_lzma2.c
@@ -0,0 +1,1171 @@
1/*
2 * LZMA2 decoder
3 *
4 * Authors: Lasse Collin <lasse.collin@tukaani.org>
5 * Igor Pavlov <http://7-zip.org/>
6 *
7 * This file has been put into the public domain.
8 * You can do whatever you want with this file.
9 */
10
11#include "xz_private.h"
12#include "xz_lzma2.h"
13
14/*
15 * Range decoder initialization eats the first five bytes of each LZMA chunk.
16 */
17#define RC_INIT_BYTES 5
18
19/*
20 * Minimum number of usable input buffer to safely decode one LZMA symbol.
21 * The worst case is that we decode 22 bits using probabilities and 26
22 * direct bits. This may decode at maximum of 20 bytes of input. However,
23 * lzma_main() does an extra normalization before returning, thus we
24 * need to put 21 here.
25 */
26#define LZMA_IN_REQUIRED 21
27
28/*
29 * Dictionary (history buffer)
30 *
31 * These are always true:
32 * start <= pos <= full <= end
33 * pos <= limit <= end
34 *
35 * In multi-call mode, also these are true:
36 * end == size
37 * size <= size_max
38 * allocated <= size
39 *
40 * Most of these variables are size_t to support single-call mode,
41 * in which the dictionary variables address the actual output
42 * buffer directly.
43 */
44struct dictionary {
45 /* Beginning of the history buffer */
46 uint8_t *buf;
47
48 /* Old position in buf (before decoding more data) */
49 size_t start;
50
51 /* Position in buf */
52 size_t pos;
53
54 /*
55 * How full dictionary is. This is used to detect corrupt input that
56 * would read beyond the beginning of the uncompressed stream.
57 */
58 size_t full;
59
60 /* Write limit; we don't write to buf[limit] or later bytes. */
61 size_t limit;
62
63 /*
64 * End of the dictionary buffer. In multi-call mode, this is
65 * the same as the dictionary size. In single-call mode, this
66 * indicates the size of the output buffer.
67 */
68 size_t end;
69
70 /*
71 * Size of the dictionary as specified in Block Header. This is used
72 * together with "full" to detect corrupt input that would make us
73 * read beyond the beginning of the uncompressed stream.
74 */
75 uint32_t size;
76
77 /*
78 * Maximum allowed dictionary size in multi-call mode.
79 * This is ignored in single-call mode.
80 */
81 uint32_t size_max;
82
83 /*
84 * Amount of memory currently allocated for the dictionary.
85 * This is used only with XZ_DYNALLOC. (With XZ_PREALLOC,
86 * size_max is always the same as the allocated size.)
87 */
88 uint32_t allocated;
89
90 /* Operation mode */
91 enum xz_mode mode;
92};
93
94/* Range decoder */
95struct rc_dec {
96 uint32_t range;
97 uint32_t code;
98
99 /*
100 * Number of initializing bytes remaining to be read
101 * by rc_read_init().
102 */
103 uint32_t init_bytes_left;
104
105 /*
106 * Buffer from which we read our input. It can be either
107 * temp.buf or the caller-provided input buffer.
108 */
109 const uint8_t *in;
110 size_t in_pos;
111 size_t in_limit;
112};
113
114/* Probabilities for a length decoder. */
115struct lzma_len_dec {
116 /* Probability of match length being at least 10 */
117 uint16_t choice;
118
119 /* Probability of match length being at least 18 */
120 uint16_t choice2;
121
122 /* Probabilities for match lengths 2-9 */
123 uint16_t low[POS_STATES_MAX][LEN_LOW_SYMBOLS];
124
125 /* Probabilities for match lengths 10-17 */
126 uint16_t mid[POS_STATES_MAX][LEN_MID_SYMBOLS];
127
128 /* Probabilities for match lengths 18-273 */
129 uint16_t high[LEN_HIGH_SYMBOLS];
130};
131
132struct lzma_dec {
133 /* Distances of latest four matches */
134 uint32_t rep0;
135 uint32_t rep1;
136 uint32_t rep2;
137 uint32_t rep3;
138
139 /* Types of the most recently seen LZMA symbols */
140 enum lzma_state state;
141
142 /*
143 * Length of a match. This is updated so that dict_repeat can
144 * be called again to finish repeating the whole match.
145 */
146 uint32_t len;
147
148 /*
149 * LZMA properties or related bit masks (number of literal
150 * context bits, a mask dervied from the number of literal
151 * position bits, and a mask dervied from the number
152 * position bits)
153 */
154 uint32_t lc;
155 uint32_t literal_pos_mask; /* (1 << lp) - 1 */
156 uint32_t pos_mask; /* (1 << pb) - 1 */
157
158 /* If 1, it's a match. Otherwise it's a single 8-bit literal. */
159 uint16_t is_match[STATES][POS_STATES_MAX];
160
161 /* If 1, it's a repeated match. The distance is one of rep0 .. rep3. */
162 uint16_t is_rep[STATES];
163
164 /*
165 * If 0, distance of a repeated match is rep0.
166 * Otherwise check is_rep1.
167 */
168 uint16_t is_rep0[STATES];
169
170 /*
171 * If 0, distance of a repeated match is rep1.
172 * Otherwise check is_rep2.
173 */
174 uint16_t is_rep1[STATES];
175
176 /* If 0, distance of a repeated match is rep2. Otherwise it is rep3. */
177 uint16_t is_rep2[STATES];
178
179 /*
180 * If 1, the repeated match has length of one byte. Otherwise
181 * the length is decoded from rep_len_decoder.
182 */
183 uint16_t is_rep0_long[STATES][POS_STATES_MAX];
184
185 /*
186 * Probability tree for the highest two bits of the match
187 * distance. There is a separate probability tree for match
188 * lengths of 2 (i.e. MATCH_LEN_MIN), 3, 4, and [5, 273].
189 */
190 uint16_t dist_slot[DIST_STATES][DIST_SLOTS];
191
192 /*
193 * Probility trees for additional bits for match distance
194 * when the distance is in the range [4, 127].
195 */
196 uint16_t dist_special[FULL_DISTANCES - DIST_MODEL_END];
197
198 /*
199 * Probability tree for the lowest four bits of a match
200 * distance that is equal to or greater than 128.
201 */
202 uint16_t dist_align[ALIGN_SIZE];
203
204 /* Length of a normal match */
205 struct lzma_len_dec match_len_dec;
206
207 /* Length of a repeated match */
208 struct lzma_len_dec rep_len_dec;
209
210 /* Probabilities of literals */
211 uint16_t literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE];
212};
213
214struct lzma2_dec {
215 /* Position in xz_dec_lzma2_run(). */
216 enum lzma2_seq {
217 SEQ_CONTROL,
218 SEQ_UNCOMPRESSED_1,
219 SEQ_UNCOMPRESSED_2,
220 SEQ_COMPRESSED_0,
221 SEQ_COMPRESSED_1,
222 SEQ_PROPERTIES,
223 SEQ_LZMA_PREPARE,
224 SEQ_LZMA_RUN,
225 SEQ_COPY
226 } sequence;
227
228 /* Next position after decoding the compressed size of the chunk. */
229 enum lzma2_seq next_sequence;
230
231 /* Uncompressed size of LZMA chunk (2 MiB at maximum) */
232 uint32_t uncompressed;
233
234 /*
235 * Compressed size of LZMA chunk or compressed/uncompressed
236 * size of uncompressed chunk (64 KiB at maximum)
237 */
238 uint32_t compressed;
239
240 /*
241 * True if dictionary reset is needed. This is false before
242 * the first chunk (LZMA or uncompressed).
243 */
244 bool need_dict_reset;
245
246 /*
247 * True if new LZMA properties are needed. This is false
248 * before the first LZMA chunk.
249 */
250 bool need_props;
251};
252
253struct xz_dec_lzma2 {
254 /*
255 * The order below is important on x86 to reduce code size and
256 * it shouldn't hurt on other platforms. Everything up to and
257 * including lzma.pos_mask are in the first 128 bytes on x86-32,
258 * which allows using smaller instructions to access those
259 * variables. On x86-64, fewer variables fit into the first 128
260 * bytes, but this is still the best order without sacrificing
261 * the readability by splitting the structures.
262 */
263 struct rc_dec rc;
264 struct dictionary dict;
265 struct lzma2_dec lzma2;
266 struct lzma_dec lzma;
267
268 /*
269 * Temporary buffer which holds small number of input bytes between
270 * decoder calls. See lzma2_lzma() for details.
271 */
272 struct {
273 uint32_t size;
274 uint8_t buf[3 * LZMA_IN_REQUIRED];
275 } temp;
276};
277
278/**************
279 * Dictionary *
280 **************/
281
282/*
283 * Reset the dictionary state. When in single-call mode, set up the beginning
284 * of the dictionary to point to the actual output buffer.
285 */
286static void dict_reset(struct dictionary *dict, struct xz_buf *b)
287{
288 if (DEC_IS_SINGLE(dict->mode)) {
289 dict->buf = b->out + b->out_pos;
290 dict->end = b->out_size - b->out_pos;
291 }
292
293 dict->start = 0;
294 dict->pos = 0;
295 dict->limit = 0;
296 dict->full = 0;
297}
298
299/* Set dictionary write limit */
300static void dict_limit(struct dictionary *dict, size_t out_max)
301{
302 if (dict->end - dict->pos <= out_max)
303 dict->limit = dict->end;
304 else
305 dict->limit = dict->pos + out_max;
306}
307
308/* Return true if at least one byte can be written into the dictionary. */
309static inline bool dict_has_space(const struct dictionary *dict)
310{
311 return dict->pos < dict->limit;
312}
313
314/*
315 * Get a byte from the dictionary at the given distance. The distance is
316 * assumed to valid, or as a special case, zero when the dictionary is
317 * still empty. This special case is needed for single-call decoding to
318 * avoid writing a '\0' to the end of the destination buffer.
319 */
320static inline uint32_t dict_get(const struct dictionary *dict, uint32_t dist)
321{
322 size_t offset = dict->pos - dist - 1;
323
324 if (dist >= dict->pos)
325 offset += dict->end;
326
327 return dict->full > 0 ? dict->buf[offset] : 0;
328}
329
330/*
331 * Put one byte into the dictionary. It is assumed that there is space for it.
332 */
333static inline void dict_put(struct dictionary *dict, uint8_t byte)
334{
335 dict->buf[dict->pos++] = byte;
336
337 if (dict->full < dict->pos)
338 dict->full = dict->pos;
339}
340
341/*
342 * Repeat given number of bytes from the given distance. If the distance is
343 * invalid, false is returned. On success, true is returned and *len is
344 * updated to indicate how many bytes were left to be repeated.
345 */
346static bool dict_repeat(struct dictionary *dict, uint32_t *len, uint32_t dist)
347{
348 size_t back;
349 uint32_t left;
350
351 if (dist >= dict->full || dist >= dict->size)
352 return false;
353
354 left = min_t(size_t, dict->limit - dict->pos, *len);
355 *len -= left;
356
357 back = dict->pos - dist - 1;
358 if (dist >= dict->pos)
359 back += dict->end;
360
361 do {
362 dict->buf[dict->pos++] = dict->buf[back++];
363 if (back == dict->end)
364 back = 0;
365 } while (--left > 0);
366
367 if (dict->full < dict->pos)
368 dict->full = dict->pos;
369
370 return true;
371}
372
373/* Copy uncompressed data as is from input to dictionary and output buffers. */
374static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b,
375 uint32_t *left)
376{
377 size_t copy_size;
378
379 while (*left > 0 && b->in_pos < b->in_size
380 && b->out_pos < b->out_size) {
381 copy_size = min(b->in_size - b->in_pos,
382 b->out_size - b->out_pos);
383 if (copy_size > dict->end - dict->pos)
384 copy_size = dict->end - dict->pos;
385 if (copy_size > *left)
386 copy_size = *left;
387
388 *left -= copy_size;
389
390 memcpy(dict->buf + dict->pos, b->in + b->in_pos, copy_size);
391 dict->pos += copy_size;
392
393 if (dict->full < dict->pos)
394 dict->full = dict->pos;
395
396 if (DEC_IS_MULTI(dict->mode)) {
397 if (dict->pos == dict->end)
398 dict->pos = 0;
399
400 memcpy(b->out + b->out_pos, b->in + b->in_pos,
401 copy_size);
402 }
403
404 dict->start = dict->pos;
405
406 b->out_pos += copy_size;
407 b->in_pos += copy_size;
408 }
409}
410
411/*
412 * Flush pending data from dictionary to b->out. It is assumed that there is
413 * enough space in b->out. This is guaranteed because caller uses dict_limit()
414 * before decoding data into the dictionary.
415 */
416static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b)
417{
418 size_t copy_size = dict->pos - dict->start;
419
420 if (DEC_IS_MULTI(dict->mode)) {
421 if (dict->pos == dict->end)
422 dict->pos = 0;
423
424 memcpy(b->out + b->out_pos, dict->buf + dict->start,
425 copy_size);
426 }
427
428 dict->start = dict->pos;
429 b->out_pos += copy_size;
430 return copy_size;
431}
432
433/*****************
434 * Range decoder *
435 *****************/
436
437/* Reset the range decoder. */
438static void rc_reset(struct rc_dec *rc)
439{
440 rc->range = (uint32_t)-1;
441 rc->code = 0;
442 rc->init_bytes_left = RC_INIT_BYTES;
443}
444
445/*
446 * Read the first five initial bytes into rc->code if they haven't been
447 * read already. (Yes, the first byte gets completely ignored.)
448 */
449static bool rc_read_init(struct rc_dec *rc, struct xz_buf *b)
450{
451 while (rc->init_bytes_left > 0) {
452 if (b->in_pos == b->in_size)
453 return false;
454
455 rc->code = (rc->code << 8) + b->in[b->in_pos++];
456 --rc->init_bytes_left;
457 }
458
459 return true;
460}
461
462/* Return true if there may not be enough input for the next decoding loop. */
463static inline bool rc_limit_exceeded(const struct rc_dec *rc)
464{
465 return rc->in_pos > rc->in_limit;
466}
467
468/*
469 * Return true if it is possible (from point of view of range decoder) that
470 * we have reached the end of the LZMA chunk.
471 */
472static inline bool rc_is_finished(const struct rc_dec *rc)
473{
474 return rc->code == 0;
475}
476
477/* Read the next input byte if needed. */
478static __always_inline void rc_normalize(struct rc_dec *rc)
479{
480 if (rc->range < RC_TOP_VALUE) {
481 rc->range <<= RC_SHIFT_BITS;
482 rc->code = (rc->code << RC_SHIFT_BITS) + rc->in[rc->in_pos++];
483 }
484}
485
486/*
487 * Decode one bit. In some versions, this function has been splitted in three
488 * functions so that the compiler is supposed to be able to more easily avoid
489 * an extra branch. In this particular version of the LZMA decoder, this
490 * doesn't seem to be a good idea (tested with GCC 3.3.6, 3.4.6, and 4.3.3
491 * on x86). Using a non-splitted version results in nicer looking code too.
492 *
493 * NOTE: This must return an int. Do not make it return a bool or the speed
494 * of the code generated by GCC 3.x decreases 10-15 %. (GCC 4.3 doesn't care,
495 * and it generates 10-20 % faster code than GCC 3.x from this file anyway.)
496 */
497static __always_inline int rc_bit(struct rc_dec *rc, uint16_t *prob)
498{
499 uint32_t bound;
500 int bit;
501
502 rc_normalize(rc);
503 bound = (rc->range >> RC_BIT_MODEL_TOTAL_BITS) * *prob;
504 if (rc->code < bound) {
505 rc->range = bound;
506 *prob += (RC_BIT_MODEL_TOTAL - *prob) >> RC_MOVE_BITS;
507 bit = 0;
508 } else {
509 rc->range -= bound;
510 rc->code -= bound;
511 *prob -= *prob >> RC_MOVE_BITS;
512 bit = 1;
513 }
514
515 return bit;
516}
517
518/* Decode a bittree starting from the most significant bit. */
519static __always_inline uint32_t rc_bittree(struct rc_dec *rc,
520 uint16_t *probs, uint32_t limit)
521{
522 uint32_t symbol = 1;
523
524 do {
525 if (rc_bit(rc, &probs[symbol]))
526 symbol = (symbol << 1) + 1;
527 else
528 symbol <<= 1;
529 } while (symbol < limit);
530
531 return symbol;
532}
533
534/* Decode a bittree starting from the least significant bit. */
535static __always_inline void rc_bittree_reverse(struct rc_dec *rc,
536 uint16_t *probs,
537 uint32_t *dest, uint32_t limit)
538{
539 uint32_t symbol = 1;
540 uint32_t i = 0;
541
542 do {
543 if (rc_bit(rc, &probs[symbol])) {
544 symbol = (symbol << 1) + 1;
545 *dest += 1 << i;
546 } else {
547 symbol <<= 1;
548 }
549 } while (++i < limit);
550}
551
552/* Decode direct bits (fixed fifty-fifty probability) */
553static inline void rc_direct(struct rc_dec *rc, uint32_t *dest, uint32_t limit)
554{
555 uint32_t mask;
556
557 do {
558 rc_normalize(rc);
559 rc->range >>= 1;
560 rc->code -= rc->range;
561 mask = (uint32_t)0 - (rc->code >> 31);
562 rc->code += rc->range & mask;
563 *dest = (*dest << 1) + (mask + 1);
564 } while (--limit > 0);
565}
566
567/********
568 * LZMA *
569 ********/
570
571/* Get pointer to literal coder probability array. */
572static uint16_t *lzma_literal_probs(struct xz_dec_lzma2 *s)
573{
574 uint32_t prev_byte = dict_get(&s->dict, 0);
575 uint32_t low = prev_byte >> (8 - s->lzma.lc);
576 uint32_t high = (s->dict.pos & s->lzma.literal_pos_mask) << s->lzma.lc;
577 return s->lzma.literal[low + high];
578}
579
580/* Decode a literal (one 8-bit byte) */
581static void lzma_literal(struct xz_dec_lzma2 *s)
582{
583 uint16_t *probs;
584 uint32_t symbol;
585 uint32_t match_byte;
586 uint32_t match_bit;
587 uint32_t offset;
588 uint32_t i;
589
590 probs = lzma_literal_probs(s);
591
592 if (lzma_state_is_literal(s->lzma.state)) {
593 symbol = rc_bittree(&s->rc, probs, 0x100);
594 } else {
595 symbol = 1;
596 match_byte = dict_get(&s->dict, s->lzma.rep0) << 1;
597 offset = 0x100;
598
599 do {
600 match_bit = match_byte & offset;
601 match_byte <<= 1;
602 i = offset + match_bit + symbol;
603
604 if (rc_bit(&s->rc, &probs[i])) {
605 symbol = (symbol << 1) + 1;
606 offset &= match_bit;
607 } else {
608 symbol <<= 1;
609 offset &= ~match_bit;
610 }
611 } while (symbol < 0x100);
612 }
613
614 dict_put(&s->dict, (uint8_t)symbol);
615 lzma_state_literal(&s->lzma.state);
616}
617
618/* Decode the length of the match into s->lzma.len. */
619static void lzma_len(struct xz_dec_lzma2 *s, struct lzma_len_dec *l,
620 uint32_t pos_state)
621{
622 uint16_t *probs;
623 uint32_t limit;
624
625 if (!rc_bit(&s->rc, &l->choice)) {
626 probs = l->low[pos_state];
627 limit = LEN_LOW_SYMBOLS;
628 s->lzma.len = MATCH_LEN_MIN;
629 } else {
630 if (!rc_bit(&s->rc, &l->choice2)) {
631 probs = l->mid[pos_state];
632 limit = LEN_MID_SYMBOLS;
633 s->lzma.len = MATCH_LEN_MIN + LEN_LOW_SYMBOLS;
634 } else {
635 probs = l->high;
636 limit = LEN_HIGH_SYMBOLS;
637 s->lzma.len = MATCH_LEN_MIN + LEN_LOW_SYMBOLS
638 + LEN_MID_SYMBOLS;
639 }
640 }
641
642 s->lzma.len += rc_bittree(&s->rc, probs, limit) - limit;
643}
644
645/* Decode a match. The distance will be stored in s->lzma.rep0. */
646static void lzma_match(struct xz_dec_lzma2 *s, uint32_t pos_state)
647{
648 uint16_t *probs;
649 uint32_t dist_slot;
650 uint32_t limit;
651
652 lzma_state_match(&s->lzma.state);
653
654 s->lzma.rep3 = s->lzma.rep2;
655 s->lzma.rep2 = s->lzma.rep1;
656 s->lzma.rep1 = s->lzma.rep0;
657
658 lzma_len(s, &s->lzma.match_len_dec, pos_state);
659
660 probs = s->lzma.dist_slot[lzma_get_dist_state(s->lzma.len)];
661 dist_slot = rc_bittree(&s->rc, probs, DIST_SLOTS) - DIST_SLOTS;
662
663 if (dist_slot < DIST_MODEL_START) {
664 s->lzma.rep0 = dist_slot;
665 } else {
666 limit = (dist_slot >> 1) - 1;
667 s->lzma.rep0 = 2 + (dist_slot & 1);
668
669 if (dist_slot < DIST_MODEL_END) {
670 s->lzma.rep0 <<= limit;
671 probs = s->lzma.dist_special + s->lzma.rep0
672 - dist_slot - 1;
673 rc_bittree_reverse(&s->rc, probs,
674 &s->lzma.rep0, limit);
675 } else {
676 rc_direct(&s->rc, &s->lzma.rep0, limit - ALIGN_BITS);
677 s->lzma.rep0 <<= ALIGN_BITS;
678 rc_bittree_reverse(&s->rc, s->lzma.dist_align,
679 &s->lzma.rep0, ALIGN_BITS);
680 }
681 }
682}
683
684/*
685 * Decode a repeated match. The distance is one of the four most recently
686 * seen matches. The distance will be stored in s->lzma.rep0.
687 */
688static void lzma_rep_match(struct xz_dec_lzma2 *s, uint32_t pos_state)
689{
690 uint32_t tmp;
691
692 if (!rc_bit(&s->rc, &s->lzma.is_rep0[s->lzma.state])) {
693 if (!rc_bit(&s->rc, &s->lzma.is_rep0_long[
694 s->lzma.state][pos_state])) {
695 lzma_state_short_rep(&s->lzma.state);
696 s->lzma.len = 1;
697 return;
698 }
699 } else {
700 if (!rc_bit(&s->rc, &s->lzma.is_rep1[s->lzma.state])) {
701 tmp = s->lzma.rep1;
702 } else {
703 if (!rc_bit(&s->rc, &s->lzma.is_rep2[s->lzma.state])) {
704 tmp = s->lzma.rep2;
705 } else {
706 tmp = s->lzma.rep3;
707 s->lzma.rep3 = s->lzma.rep2;
708 }
709
710 s->lzma.rep2 = s->lzma.rep1;
711 }
712
713 s->lzma.rep1 = s->lzma.rep0;
714 s->lzma.rep0 = tmp;
715 }
716
717 lzma_state_long_rep(&s->lzma.state);
718 lzma_len(s, &s->lzma.rep_len_dec, pos_state);
719}
720
721/* LZMA decoder core */
722static bool lzma_main(struct xz_dec_lzma2 *s)
723{
724 uint32_t pos_state;
725
726 /*
727 * If the dictionary was reached during the previous call, try to
728 * finish the possibly pending repeat in the dictionary.
729 */
730 if (dict_has_space(&s->dict) && s->lzma.len > 0)
731 dict_repeat(&s->dict, &s->lzma.len, s->lzma.rep0);
732
733 /*
734 * Decode more LZMA symbols. One iteration may consume up to
735 * LZMA_IN_REQUIRED - 1 bytes.
736 */
737 while (dict_has_space(&s->dict) && !rc_limit_exceeded(&s->rc)) {
738 pos_state = s->dict.pos & s->lzma.pos_mask;
739
740 if (!rc_bit(&s->rc, &s->lzma.is_match[
741 s->lzma.state][pos_state])) {
742 lzma_literal(s);
743 } else {
744 if (rc_bit(&s->rc, &s->lzma.is_rep[s->lzma.state]))
745 lzma_rep_match(s, pos_state);
746 else
747 lzma_match(s, pos_state);
748
749 if (!dict_repeat(&s->dict, &s->lzma.len, s->lzma.rep0))
750 return false;
751 }
752 }
753
754 /*
755 * Having the range decoder always normalized when we are outside
756 * this function makes it easier to correctly handle end of the chunk.
757 */
758 rc_normalize(&s->rc);
759
760 return true;
761}
762
763/*
764 * Reset the LZMA decoder and range decoder state. Dictionary is nore reset
765 * here, because LZMA state may be reset without resetting the dictionary.
766 */
767static void lzma_reset(struct xz_dec_lzma2 *s)
768{
769 uint16_t *probs;
770 size_t i;
771
772 s->lzma.state = STATE_LIT_LIT;
773 s->lzma.rep0 = 0;
774 s->lzma.rep1 = 0;
775 s->lzma.rep2 = 0;
776 s->lzma.rep3 = 0;
777
778 /*
779 * All probabilities are initialized to the same value. This hack
780 * makes the code smaller by avoiding a separate loop for each
781 * probability array.
782 *
783 * This could be optimized so that only that part of literal
784 * probabilities that are actually required. In the common case
785 * we would write 12 KiB less.
786 */
787 probs = s->lzma.is_match[0];
788 for (i = 0; i < PROBS_TOTAL; ++i)
789 probs[i] = RC_BIT_MODEL_TOTAL / 2;
790
791 rc_reset(&s->rc);
792}
793
794/*
795 * Decode and validate LZMA properties (lc/lp/pb) and calculate the bit masks
796 * from the decoded lp and pb values. On success, the LZMA decoder state is
797 * reset and true is returned.
798 */
799static bool lzma_props(struct xz_dec_lzma2 *s, uint8_t props)
800{
801 if (props > (4 * 5 + 4) * 9 + 8)
802 return false;
803
804 s->lzma.pos_mask = 0;
805 while (props >= 9 * 5) {
806 props -= 9 * 5;
807 ++s->lzma.pos_mask;
808 }
809
810 s->lzma.pos_mask = (1 << s->lzma.pos_mask) - 1;
811
812 s->lzma.literal_pos_mask = 0;
813 while (props >= 9) {
814 props -= 9;
815 ++s->lzma.literal_pos_mask;
816 }
817
818 s->lzma.lc = props;
819
820 if (s->lzma.lc + s->lzma.literal_pos_mask > 4)
821 return false;
822
823 s->lzma.literal_pos_mask = (1 << s->lzma.literal_pos_mask) - 1;
824
825 lzma_reset(s);
826
827 return true;
828}
829
830/*********
831 * LZMA2 *
832 *********/
833
834/*
835 * The LZMA decoder assumes that if the input limit (s->rc.in_limit) hasn't
836 * been exceeded, it is safe to read up to LZMA_IN_REQUIRED bytes. This
837 * wrapper function takes care of making the LZMA decoder's assumption safe.
838 *
839 * As long as there is plenty of input left to be decoded in the current LZMA
840 * chunk, we decode directly from the caller-supplied input buffer until
841 * there's LZMA_IN_REQUIRED bytes left. Those remaining bytes are copied into
842 * s->temp.buf, which (hopefully) gets filled on the next call to this
843 * function. We decode a few bytes from the temporary buffer so that we can
844 * continue decoding from the caller-supplied input buffer again.
845 */
846static bool lzma2_lzma(struct xz_dec_lzma2 *s, struct xz_buf *b)
847{
848 size_t in_avail;
849 uint32_t tmp;
850
851 in_avail = b->in_size - b->in_pos;
852 if (s->temp.size > 0 || s->lzma2.compressed == 0) {
853 tmp = 2 * LZMA_IN_REQUIRED - s->temp.size;
854 if (tmp > s->lzma2.compressed - s->temp.size)
855 tmp = s->lzma2.compressed - s->temp.size;
856 if (tmp > in_avail)
857 tmp = in_avail;
858
859 memcpy(s->temp.buf + s->temp.size, b->in + b->in_pos, tmp);
860
861 if (s->temp.size + tmp == s->lzma2.compressed) {
862 memzero(s->temp.buf + s->temp.size + tmp,
863 sizeof(s->temp.buf)
864 - s->temp.size - tmp);
865 s->rc.in_limit = s->temp.size + tmp;
866 } else if (s->temp.size + tmp < LZMA_IN_REQUIRED) {
867 s->temp.size += tmp;
868 b->in_pos += tmp;
869 return true;
870 } else {
871 s->rc.in_limit = s->temp.size + tmp - LZMA_IN_REQUIRED;
872 }
873
874 s->rc.in = s->temp.buf;
875 s->rc.in_pos = 0;
876
877 if (!lzma_main(s) || s->rc.in_pos > s->temp.size + tmp)
878 return false;
879
880 s->lzma2.compressed -= s->rc.in_pos;
881
882 if (s->rc.in_pos < s->temp.size) {
883 s->temp.size -= s->rc.in_pos;
884 memmove(s->temp.buf, s->temp.buf + s->rc.in_pos,
885 s->temp.size);
886 return true;
887 }
888
889 b->in_pos += s->rc.in_pos - s->temp.size;
890 s->temp.size = 0;
891 }
892
893 in_avail = b->in_size - b->in_pos;
894 if (in_avail >= LZMA_IN_REQUIRED) {
895 s->rc.in = b->in;
896 s->rc.in_pos = b->in_pos;
897
898 if (in_avail >= s->lzma2.compressed + LZMA_IN_REQUIRED)
899 s->rc.in_limit = b->in_pos + s->lzma2.compressed;
900 else
901 s->rc.in_limit = b->in_size - LZMA_IN_REQUIRED;
902
903 if (!lzma_main(s))
904 return false;
905
906 in_avail = s->rc.in_pos - b->in_pos;
907 if (in_avail > s->lzma2.compressed)
908 return false;
909
910 s->lzma2.compressed -= in_avail;
911 b->in_pos = s->rc.in_pos;
912 }
913
914 in_avail = b->in_size - b->in_pos;
915 if (in_avail < LZMA_IN_REQUIRED) {
916 if (in_avail > s->lzma2.compressed)
917 in_avail = s->lzma2.compressed;
918
919 memcpy(s->temp.buf, b->in + b->in_pos, in_avail);
920 s->temp.size = in_avail;
921 b->in_pos += in_avail;
922 }
923
924 return true;
925}
926
927/*
928 * Take care of the LZMA2 control layer, and forward the job of actual LZMA
929 * decoding or copying of uncompressed chunks to other functions.
930 */
931XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s,
932 struct xz_buf *b)
933{
934 uint32_t tmp;
935
936 while (b->in_pos < b->in_size || s->lzma2.sequence == SEQ_LZMA_RUN) {
937 switch (s->lzma2.sequence) {
938 case SEQ_CONTROL:
939 /*
940 * LZMA2 control byte
941 *
942 * Exact values:
943 * 0x00 End marker
944 * 0x01 Dictionary reset followed by
945 * an uncompressed chunk
946 * 0x02 Uncompressed chunk (no dictionary reset)
947 *
948 * Highest three bits (s->control & 0xE0):
949 * 0xE0 Dictionary reset, new properties and state
950 * reset, followed by LZMA compressed chunk
951 * 0xC0 New properties and state reset, followed
952 * by LZMA compressed chunk (no dictionary
953 * reset)
954 * 0xA0 State reset using old properties,
955 * followed by LZMA compressed chunk (no
956 * dictionary reset)
957 * 0x80 LZMA chunk (no dictionary or state reset)
958 *
959 * For LZMA compressed chunks, the lowest five bits
960 * (s->control & 1F) are the highest bits of the
961 * uncompressed size (bits 16-20).
962 *
963 * A new LZMA2 stream must begin with a dictionary
964 * reset. The first LZMA chunk must set new
965 * properties and reset the LZMA state.
966 *
967 * Values that don't match anything described above
968 * are invalid and we return XZ_DATA_ERROR.
969 */
970 tmp = b->in[b->in_pos++];
971
972 if (tmp == 0x00)
973 return XZ_STREAM_END;
974
975 if (tmp >= 0xE0 || tmp == 0x01) {
976 s->lzma2.need_props = true;
977 s->lzma2.need_dict_reset = false;
978 dict_reset(&s->dict, b);
979 } else if (s->lzma2.need_dict_reset) {
980 return XZ_DATA_ERROR;
981 }
982
983 if (tmp >= 0x80) {
984 s->lzma2.uncompressed = (tmp & 0x1F) << 16;
985 s->lzma2.sequence = SEQ_UNCOMPRESSED_1;
986
987 if (tmp >= 0xC0) {
988 /*
989 * When there are new properties,
990 * state reset is done at
991 * SEQ_PROPERTIES.
992 */
993 s->lzma2.need_props = false;
994 s->lzma2.next_sequence
995 = SEQ_PROPERTIES;
996
997 } else if (s->lzma2.need_props) {
998 return XZ_DATA_ERROR;
999
1000 } else {
1001 s->lzma2.next_sequence
1002 = SEQ_LZMA_PREPARE;
1003 if (tmp >= 0xA0)
1004 lzma_reset(s);
1005 }
1006 } else {
1007 if (tmp > 0x02)
1008 return XZ_DATA_ERROR;
1009
1010 s->lzma2.sequence = SEQ_COMPRESSED_0;
1011 s->lzma2.next_sequence = SEQ_COPY;
1012 }
1013
1014 break;
1015
1016 case SEQ_UNCOMPRESSED_1:
1017 s->lzma2.uncompressed
1018 += (uint32_t)b->in[b->in_pos++] << 8;
1019 s->lzma2.sequence = SEQ_UNCOMPRESSED_2;
1020 break;
1021
1022 case SEQ_UNCOMPRESSED_2:
1023 s->lzma2.uncompressed
1024 += (uint32_t)b->in[b->in_pos++] + 1;
1025 s->lzma2.sequence = SEQ_COMPRESSED_0;
1026 break;
1027
1028 case SEQ_COMPRESSED_0:
1029 s->lzma2.compressed
1030 = (uint32_t)b->in[b->in_pos++] << 8;
1031 s->lzma2.sequence = SEQ_COMPRESSED_1;
1032 break;
1033
1034 case SEQ_COMPRESSED_1:
1035 s->lzma2.compressed
1036 += (uint32_t)b->in[b->in_pos++] + 1;
1037 s->lzma2.sequence = s->lzma2.next_sequence;
1038 break;
1039
1040 case SEQ_PROPERTIES:
1041 if (!lzma_props(s, b->in[b->in_pos++]))
1042 return XZ_DATA_ERROR;
1043
1044 s->lzma2.sequence = SEQ_LZMA_PREPARE;
1045
1046 case SEQ_LZMA_PREPARE:
1047 if (s->lzma2.compressed < RC_INIT_BYTES)
1048 return XZ_DATA_ERROR;
1049
1050 if (!rc_read_init(&s->rc, b))
1051 return XZ_OK;
1052
1053 s->lzma2.compressed -= RC_INIT_BYTES;
1054 s->lzma2.sequence = SEQ_LZMA_RUN;
1055
1056 case SEQ_LZMA_RUN:
1057 /*
1058 * Set dictionary limit to indicate how much we want
1059 * to be encoded at maximum. Decode new data into the
1060 * dictionary. Flush the new data from dictionary to
1061 * b->out. Check if we finished decoding this chunk.
1062 * In case the dictionary got full but we didn't fill
1063 * the output buffer yet, we may run this loop
1064 * multiple times without changing s->lzma2.sequence.
1065 */
1066 dict_limit(&s->dict, min_t(size_t,
1067 b->out_size - b->out_pos,
1068 s->lzma2.uncompressed));
1069 if (!lzma2_lzma(s, b))
1070 return XZ_DATA_ERROR;
1071
1072 s->lzma2.uncompressed -= dict_flush(&s->dict, b);
1073
1074 if (s->lzma2.uncompressed == 0) {
1075 if (s->lzma2.compressed > 0 || s->lzma.len > 0
1076 || !rc_is_finished(&s->rc))
1077 return XZ_DATA_ERROR;
1078
1079 rc_reset(&s->rc);
1080 s->lzma2.sequence = SEQ_CONTROL;
1081
1082 } else if (b->out_pos == b->out_size
1083 || (b->in_pos == b->in_size
1084 && s->temp.size
1085 < s->lzma2.compressed)) {
1086 return XZ_OK;
1087 }
1088
1089 break;
1090
1091 case SEQ_COPY:
1092 dict_uncompressed(&s->dict, b, &s->lzma2.compressed);
1093 if (s->lzma2.compressed > 0)
1094 return XZ_OK;
1095
1096 s->lzma2.sequence = SEQ_CONTROL;
1097 break;
1098 }
1099 }
1100
1101 return XZ_OK;
1102}
1103
1104XZ_EXTERN struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode,
1105 uint32_t dict_max)
1106{
1107 struct xz_dec_lzma2 *s = kmalloc(sizeof(*s), GFP_KERNEL);
1108 if (s == NULL)
1109 return NULL;
1110
1111 s->dict.mode = mode;
1112 s->dict.size_max = dict_max;
1113
1114 if (DEC_IS_PREALLOC(mode)) {
1115 s->dict.buf = vmalloc(dict_max);
1116 if (s->dict.buf == NULL) {
1117 kfree(s);
1118 return NULL;
1119 }
1120 } else if (DEC_IS_DYNALLOC(mode)) {
1121 s->dict.buf = NULL;
1122 s->dict.allocated = 0;
1123 }
1124
1125 return s;
1126}
1127
1128XZ_EXTERN enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, uint8_t props)
1129{
1130 /* This limits dictionary size to 3 GiB to keep parsing simpler. */
1131 if (props > 39)
1132 return XZ_OPTIONS_ERROR;
1133
1134 s->dict.size = 2 + (props & 1);
1135 s->dict.size <<= (props >> 1) + 11;
1136
1137 if (DEC_IS_MULTI(s->dict.mode)) {
1138 if (s->dict.size > s->dict.size_max)
1139 return XZ_MEMLIMIT_ERROR;
1140
1141 s->dict.end = s->dict.size;
1142
1143 if (DEC_IS_DYNALLOC(s->dict.mode)) {
1144 if (s->dict.allocated < s->dict.size) {
1145 vfree(s->dict.buf);
1146 s->dict.buf = vmalloc(s->dict.size);
1147 if (s->dict.buf == NULL) {
1148 s->dict.allocated = 0;
1149 return XZ_MEM_ERROR;
1150 }
1151 }
1152 }
1153 }
1154
1155 s->lzma.len = 0;
1156
1157 s->lzma2.sequence = SEQ_CONTROL;
1158 s->lzma2.need_dict_reset = true;
1159
1160 s->temp.size = 0;
1161
1162 return XZ_OK;
1163}
1164
1165XZ_EXTERN void xz_dec_lzma2_end(struct xz_dec_lzma2 *s)
1166{
1167 if (DEC_IS_MULTI(s->dict.mode))
1168 vfree(s->dict.buf);
1169
1170 kfree(s);
1171}
diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c
new file mode 100644
index 000000000000..ac809b1e64f7
--- /dev/null
+++ b/lib/xz/xz_dec_stream.c
@@ -0,0 +1,821 @@
1/*
2 * .xz Stream decoder
3 *
4 * Author: Lasse Collin <lasse.collin@tukaani.org>
5 *
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
8 */
9
10#include "xz_private.h"
11#include "xz_stream.h"
12
13/* Hash used to validate the Index field */
14struct xz_dec_hash {
15 vli_type unpadded;
16 vli_type uncompressed;
17 uint32_t crc32;
18};
19
20struct xz_dec {
21 /* Position in dec_main() */
22 enum {
23 SEQ_STREAM_HEADER,
24 SEQ_BLOCK_START,
25 SEQ_BLOCK_HEADER,
26 SEQ_BLOCK_UNCOMPRESS,
27 SEQ_BLOCK_PADDING,
28 SEQ_BLOCK_CHECK,
29 SEQ_INDEX,
30 SEQ_INDEX_PADDING,
31 SEQ_INDEX_CRC32,
32 SEQ_STREAM_FOOTER
33 } sequence;
34
35 /* Position in variable-length integers and Check fields */
36 uint32_t pos;
37
38 /* Variable-length integer decoded by dec_vli() */
39 vli_type vli;
40
41 /* Saved in_pos and out_pos */
42 size_t in_start;
43 size_t out_start;
44
45 /* CRC32 value in Block or Index */
46 uint32_t crc32;
47
48 /* Type of the integrity check calculated from uncompressed data */
49 enum xz_check check_type;
50
51 /* Operation mode */
52 enum xz_mode mode;
53
54 /*
55 * True if the next call to xz_dec_run() is allowed to return
56 * XZ_BUF_ERROR.
57 */
58 bool allow_buf_error;
59
60 /* Information stored in Block Header */
61 struct {
62 /*
63 * Value stored in the Compressed Size field, or
64 * VLI_UNKNOWN if Compressed Size is not present.
65 */
66 vli_type compressed;
67
68 /*
69 * Value stored in the Uncompressed Size field, or
70 * VLI_UNKNOWN if Uncompressed Size is not present.
71 */
72 vli_type uncompressed;
73
74 /* Size of the Block Header field */
75 uint32_t size;
76 } block_header;
77
78 /* Information collected when decoding Blocks */
79 struct {
80 /* Observed compressed size of the current Block */
81 vli_type compressed;
82
83 /* Observed uncompressed size of the current Block */
84 vli_type uncompressed;
85
86 /* Number of Blocks decoded so far */
87 vli_type count;
88
89 /*
90 * Hash calculated from the Block sizes. This is used to
91 * validate the Index field.
92 */
93 struct xz_dec_hash hash;
94 } block;
95
96 /* Variables needed when verifying the Index field */
97 struct {
98 /* Position in dec_index() */
99 enum {
100 SEQ_INDEX_COUNT,
101 SEQ_INDEX_UNPADDED,
102 SEQ_INDEX_UNCOMPRESSED
103 } sequence;
104
105 /* Size of the Index in bytes */
106 vli_type size;
107
108 /* Number of Records (matches block.count in valid files) */
109 vli_type count;
110
111 /*
112 * Hash calculated from the Records (matches block.hash in
113 * valid files).
114 */
115 struct xz_dec_hash hash;
116 } index;
117
118 /*
119 * Temporary buffer needed to hold Stream Header, Block Header,
120 * and Stream Footer. The Block Header is the biggest (1 KiB)
121 * so we reserve space according to that. buf[] has to be aligned
122 * to a multiple of four bytes; the size_t variables before it
123 * should guarantee this.
124 */
125 struct {
126 size_t pos;
127 size_t size;
128 uint8_t buf[1024];
129 } temp;
130
131 struct xz_dec_lzma2 *lzma2;
132
133#ifdef XZ_DEC_BCJ
134 struct xz_dec_bcj *bcj;
135 bool bcj_active;
136#endif
137};
138
139#ifdef XZ_DEC_ANY_CHECK
140/* Sizes of the Check field with different Check IDs */
141static const uint8_t check_sizes[16] = {
142 0,
143 4, 4, 4,
144 8, 8, 8,
145 16, 16, 16,
146 32, 32, 32,
147 64, 64, 64
148};
149#endif
150
151/*
152 * Fill s->temp by copying data starting from b->in[b->in_pos]. Caller
153 * must have set s->temp.pos to indicate how much data we are supposed
154 * to copy into s->temp.buf. Return true once s->temp.pos has reached
155 * s->temp.size.
156 */
157static bool fill_temp(struct xz_dec *s, struct xz_buf *b)
158{
159 size_t copy_size = min_t(size_t,
160 b->in_size - b->in_pos, s->temp.size - s->temp.pos);
161
162 memcpy(s->temp.buf + s->temp.pos, b->in + b->in_pos, copy_size);
163 b->in_pos += copy_size;
164 s->temp.pos += copy_size;
165
166 if (s->temp.pos == s->temp.size) {
167 s->temp.pos = 0;
168 return true;
169 }
170
171 return false;
172}
173
174/* Decode a variable-length integer (little-endian base-128 encoding) */
175static enum xz_ret dec_vli(struct xz_dec *s, const uint8_t *in,
176 size_t *in_pos, size_t in_size)
177{
178 uint8_t byte;
179
180 if (s->pos == 0)
181 s->vli = 0;
182
183 while (*in_pos < in_size) {
184 byte = in[*in_pos];
185 ++*in_pos;
186
187 s->vli |= (vli_type)(byte & 0x7F) << s->pos;
188
189 if ((byte & 0x80) == 0) {
190 /* Don't allow non-minimal encodings. */
191 if (byte == 0 && s->pos != 0)
192 return XZ_DATA_ERROR;
193
194 s->pos = 0;
195 return XZ_STREAM_END;
196 }
197
198 s->pos += 7;
199 if (s->pos == 7 * VLI_BYTES_MAX)
200 return XZ_DATA_ERROR;
201 }
202
203 return XZ_OK;
204}
205
206/*
207 * Decode the Compressed Data field from a Block. Update and validate
208 * the observed compressed and uncompressed sizes of the Block so that
209 * they don't exceed the values possibly stored in the Block Header
210 * (validation assumes that no integer overflow occurs, since vli_type
211 * is normally uint64_t). Update the CRC32 if presence of the CRC32
212 * field was indicated in Stream Header.
213 *
214 * Once the decoding is finished, validate that the observed sizes match
215 * the sizes possibly stored in the Block Header. Update the hash and
216 * Block count, which are later used to validate the Index field.
217 */
218static enum xz_ret dec_block(struct xz_dec *s, struct xz_buf *b)
219{
220 enum xz_ret ret;
221
222 s->in_start = b->in_pos;
223 s->out_start = b->out_pos;
224
225#ifdef XZ_DEC_BCJ
226 if (s->bcj_active)
227 ret = xz_dec_bcj_run(s->bcj, s->lzma2, b);
228 else
229#endif
230 ret = xz_dec_lzma2_run(s->lzma2, b);
231
232 s->block.compressed += b->in_pos - s->in_start;
233 s->block.uncompressed += b->out_pos - s->out_start;
234
235 /*
236 * There is no need to separately check for VLI_UNKNOWN, since
237 * the observed sizes are always smaller than VLI_UNKNOWN.
238 */
239 if (s->block.compressed > s->block_header.compressed
240 || s->block.uncompressed
241 > s->block_header.uncompressed)
242 return XZ_DATA_ERROR;
243
244 if (s->check_type == XZ_CHECK_CRC32)
245 s->crc32 = xz_crc32(b->out + s->out_start,
246 b->out_pos - s->out_start, s->crc32);
247
248 if (ret == XZ_STREAM_END) {
249 if (s->block_header.compressed != VLI_UNKNOWN
250 && s->block_header.compressed
251 != s->block.compressed)
252 return XZ_DATA_ERROR;
253
254 if (s->block_header.uncompressed != VLI_UNKNOWN
255 && s->block_header.uncompressed
256 != s->block.uncompressed)
257 return XZ_DATA_ERROR;
258
259 s->block.hash.unpadded += s->block_header.size
260 + s->block.compressed;
261
262#ifdef XZ_DEC_ANY_CHECK
263 s->block.hash.unpadded += check_sizes[s->check_type];
264#else
265 if (s->check_type == XZ_CHECK_CRC32)
266 s->block.hash.unpadded += 4;
267#endif
268
269 s->block.hash.uncompressed += s->block.uncompressed;
270 s->block.hash.crc32 = xz_crc32(
271 (const uint8_t *)&s->block.hash,
272 sizeof(s->block.hash), s->block.hash.crc32);
273
274 ++s->block.count;
275 }
276
277 return ret;
278}
279
280/* Update the Index size and the CRC32 value. */
281static void index_update(struct xz_dec *s, const struct xz_buf *b)
282{
283 size_t in_used = b->in_pos - s->in_start;
284 s->index.size += in_used;
285 s->crc32 = xz_crc32(b->in + s->in_start, in_used, s->crc32);
286}
287
288/*
289 * Decode the Number of Records, Unpadded Size, and Uncompressed Size
290 * fields from the Index field. That is, Index Padding and CRC32 are not
291 * decoded by this function.
292 *
293 * This can return XZ_OK (more input needed), XZ_STREAM_END (everything
294 * successfully decoded), or XZ_DATA_ERROR (input is corrupt).
295 */
296static enum xz_ret dec_index(struct xz_dec *s, struct xz_buf *b)
297{
298 enum xz_ret ret;
299
300 do {
301 ret = dec_vli(s, b->in, &b->in_pos, b->in_size);
302 if (ret != XZ_STREAM_END) {
303 index_update(s, b);
304 return ret;
305 }
306
307 switch (s->index.sequence) {
308 case SEQ_INDEX_COUNT:
309 s->index.count = s->vli;
310
311 /*
312 * Validate that the Number of Records field
313 * indicates the same number of Records as
314 * there were Blocks in the Stream.
315 */
316 if (s->index.count != s->block.count)
317 return XZ_DATA_ERROR;
318
319 s->index.sequence = SEQ_INDEX_UNPADDED;
320 break;
321
322 case SEQ_INDEX_UNPADDED:
323 s->index.hash.unpadded += s->vli;
324 s->index.sequence = SEQ_INDEX_UNCOMPRESSED;
325 break;
326
327 case SEQ_INDEX_UNCOMPRESSED:
328 s->index.hash.uncompressed += s->vli;
329 s->index.hash.crc32 = xz_crc32(
330 (const uint8_t *)&s->index.hash,
331 sizeof(s->index.hash),
332 s->index.hash.crc32);
333 --s->index.count;
334 s->index.sequence = SEQ_INDEX_UNPADDED;
335 break;
336 }
337 } while (s->index.count > 0);
338
339 return XZ_STREAM_END;
340}
341
342/*
343 * Validate that the next four input bytes match the value of s->crc32.
344 * s->pos must be zero when starting to validate the first byte.
345 */
346static enum xz_ret crc32_validate(struct xz_dec *s, struct xz_buf *b)
347{
348 do {
349 if (b->in_pos == b->in_size)
350 return XZ_OK;
351
352 if (((s->crc32 >> s->pos) & 0xFF) != b->in[b->in_pos++])
353 return XZ_DATA_ERROR;
354
355 s->pos += 8;
356
357 } while (s->pos < 32);
358
359 s->crc32 = 0;
360 s->pos = 0;
361
362 return XZ_STREAM_END;
363}
364
365#ifdef XZ_DEC_ANY_CHECK
366/*
367 * Skip over the Check field when the Check ID is not supported.
368 * Returns true once the whole Check field has been skipped over.
369 */
370static bool check_skip(struct xz_dec *s, struct xz_buf *b)
371{
372 while (s->pos < check_sizes[s->check_type]) {
373 if (b->in_pos == b->in_size)
374 return false;
375
376 ++b->in_pos;
377 ++s->pos;
378 }
379
380 s->pos = 0;
381
382 return true;
383}
384#endif
385
386/* Decode the Stream Header field (the first 12 bytes of the .xz Stream). */
387static enum xz_ret dec_stream_header(struct xz_dec *s)
388{
389 if (!memeq(s->temp.buf, HEADER_MAGIC, HEADER_MAGIC_SIZE))
390 return XZ_FORMAT_ERROR;
391
392 if (xz_crc32(s->temp.buf + HEADER_MAGIC_SIZE, 2, 0)
393 != get_le32(s->temp.buf + HEADER_MAGIC_SIZE + 2))
394 return XZ_DATA_ERROR;
395
396 if (s->temp.buf[HEADER_MAGIC_SIZE] != 0)
397 return XZ_OPTIONS_ERROR;
398
399 /*
400 * Of integrity checks, we support only none (Check ID = 0) and
401 * CRC32 (Check ID = 1). However, if XZ_DEC_ANY_CHECK is defined,
402 * we will accept other check types too, but then the check won't
403 * be verified and a warning (XZ_UNSUPPORTED_CHECK) will be given.
404 */
405 s->check_type = s->temp.buf[HEADER_MAGIC_SIZE + 1];
406
407#ifdef XZ_DEC_ANY_CHECK
408 if (s->check_type > XZ_CHECK_MAX)
409 return XZ_OPTIONS_ERROR;
410
411 if (s->check_type > XZ_CHECK_CRC32)
412 return XZ_UNSUPPORTED_CHECK;
413#else
414 if (s->check_type > XZ_CHECK_CRC32)
415 return XZ_OPTIONS_ERROR;
416#endif
417
418 return XZ_OK;
419}
420
421/* Decode the Stream Footer field (the last 12 bytes of the .xz Stream) */
422static enum xz_ret dec_stream_footer(struct xz_dec *s)
423{
424 if (!memeq(s->temp.buf + 10, FOOTER_MAGIC, FOOTER_MAGIC_SIZE))
425 return XZ_DATA_ERROR;
426
427 if (xz_crc32(s->temp.buf + 4, 6, 0) != get_le32(s->temp.buf))
428 return XZ_DATA_ERROR;
429
430 /*
431 * Validate Backward Size. Note that we never added the size of the
432 * Index CRC32 field to s->index.size, thus we use s->index.size / 4
433 * instead of s->index.size / 4 - 1.
434 */
435 if ((s->index.size >> 2) != get_le32(s->temp.buf + 4))
436 return XZ_DATA_ERROR;
437
438 if (s->temp.buf[8] != 0 || s->temp.buf[9] != s->check_type)
439 return XZ_DATA_ERROR;
440
441 /*
442 * Use XZ_STREAM_END instead of XZ_OK to be more convenient
443 * for the caller.
444 */
445 return XZ_STREAM_END;
446}
447
448/* Decode the Block Header and initialize the filter chain. */
449static enum xz_ret dec_block_header(struct xz_dec *s)
450{
451 enum xz_ret ret;
452
453 /*
454 * Validate the CRC32. We know that the temp buffer is at least
455 * eight bytes so this is safe.
456 */
457 s->temp.size -= 4;
458 if (xz_crc32(s->temp.buf, s->temp.size, 0)
459 != get_le32(s->temp.buf + s->temp.size))
460 return XZ_DATA_ERROR;
461
462 s->temp.pos = 2;
463
464 /*
465 * Catch unsupported Block Flags. We support only one or two filters
466 * in the chain, so we catch that with the same test.
467 */
468#ifdef XZ_DEC_BCJ
469 if (s->temp.buf[1] & 0x3E)
470#else
471 if (s->temp.buf[1] & 0x3F)
472#endif
473 return XZ_OPTIONS_ERROR;
474
475 /* Compressed Size */
476 if (s->temp.buf[1] & 0x40) {
477 if (dec_vli(s, s->temp.buf, &s->temp.pos, s->temp.size)
478 != XZ_STREAM_END)
479 return XZ_DATA_ERROR;
480
481 s->block_header.compressed = s->vli;
482 } else {
483 s->block_header.compressed = VLI_UNKNOWN;
484 }
485
486 /* Uncompressed Size */
487 if (s->temp.buf[1] & 0x80) {
488 if (dec_vli(s, s->temp.buf, &s->temp.pos, s->temp.size)
489 != XZ_STREAM_END)
490 return XZ_DATA_ERROR;
491
492 s->block_header.uncompressed = s->vli;
493 } else {
494 s->block_header.uncompressed = VLI_UNKNOWN;
495 }
496
497#ifdef XZ_DEC_BCJ
498 /* If there are two filters, the first one must be a BCJ filter. */
499 s->bcj_active = s->temp.buf[1] & 0x01;
500 if (s->bcj_active) {
501 if (s->temp.size - s->temp.pos < 2)
502 return XZ_OPTIONS_ERROR;
503
504 ret = xz_dec_bcj_reset(s->bcj, s->temp.buf[s->temp.pos++]);
505 if (ret != XZ_OK)
506 return ret;
507
508 /*
509 * We don't support custom start offset,
510 * so Size of Properties must be zero.
511 */
512 if (s->temp.buf[s->temp.pos++] != 0x00)
513 return XZ_OPTIONS_ERROR;
514 }
515#endif
516
517 /* Valid Filter Flags always take at least two bytes. */
518 if (s->temp.size - s->temp.pos < 2)
519 return XZ_DATA_ERROR;
520
521 /* Filter ID = LZMA2 */
522 if (s->temp.buf[s->temp.pos++] != 0x21)
523 return XZ_OPTIONS_ERROR;
524
525 /* Size of Properties = 1-byte Filter Properties */
526 if (s->temp.buf[s->temp.pos++] != 0x01)
527 return XZ_OPTIONS_ERROR;
528
529 /* Filter Properties contains LZMA2 dictionary size. */
530 if (s->temp.size - s->temp.pos < 1)
531 return XZ_DATA_ERROR;
532
533 ret = xz_dec_lzma2_reset(s->lzma2, s->temp.buf[s->temp.pos++]);
534 if (ret != XZ_OK)
535 return ret;
536
537 /* The rest must be Header Padding. */
538 while (s->temp.pos < s->temp.size)
539 if (s->temp.buf[s->temp.pos++] != 0x00)
540 return XZ_OPTIONS_ERROR;
541
542 s->temp.pos = 0;
543 s->block.compressed = 0;
544 s->block.uncompressed = 0;
545
546 return XZ_OK;
547}
548
549static enum xz_ret dec_main(struct xz_dec *s, struct xz_buf *b)
550{
551 enum xz_ret ret;
552
553 /*
554 * Store the start position for the case when we are in the middle
555 * of the Index field.
556 */
557 s->in_start = b->in_pos;
558
559 while (true) {
560 switch (s->sequence) {
561 case SEQ_STREAM_HEADER:
562 /*
563 * Stream Header is copied to s->temp, and then
564 * decoded from there. This way if the caller
565 * gives us only little input at a time, we can
566 * still keep the Stream Header decoding code
567 * simple. Similar approach is used in many places
568 * in this file.
569 */
570 if (!fill_temp(s, b))
571 return XZ_OK;
572
573 /*
574 * If dec_stream_header() returns
575 * XZ_UNSUPPORTED_CHECK, it is still possible
576 * to continue decoding if working in multi-call
577 * mode. Thus, update s->sequence before calling
578 * dec_stream_header().
579 */
580 s->sequence = SEQ_BLOCK_START;
581
582 ret = dec_stream_header(s);
583 if (ret != XZ_OK)
584 return ret;
585
586 case SEQ_BLOCK_START:
587 /* We need one byte of input to continue. */
588 if (b->in_pos == b->in_size)
589 return XZ_OK;
590
591 /* See if this is the beginning of the Index field. */
592 if (b->in[b->in_pos] == 0) {
593 s->in_start = b->in_pos++;
594 s->sequence = SEQ_INDEX;
595 break;
596 }
597
598 /*
599 * Calculate the size of the Block Header and
600 * prepare to decode it.
601 */
602 s->block_header.size
603 = ((uint32_t)b->in[b->in_pos] + 1) * 4;
604
605 s->temp.size = s->block_header.size;
606 s->temp.pos = 0;
607 s->sequence = SEQ_BLOCK_HEADER;
608
609 case SEQ_BLOCK_HEADER:
610 if (!fill_temp(s, b))
611 return XZ_OK;
612
613 ret = dec_block_header(s);
614 if (ret != XZ_OK)
615 return ret;
616
617 s->sequence = SEQ_BLOCK_UNCOMPRESS;
618
619 case SEQ_BLOCK_UNCOMPRESS:
620 ret = dec_block(s, b);
621 if (ret != XZ_STREAM_END)
622 return ret;
623
624 s->sequence = SEQ_BLOCK_PADDING;
625
626 case SEQ_BLOCK_PADDING:
627 /*
628 * Size of Compressed Data + Block Padding
629 * must be a multiple of four. We don't need
630 * s->block.compressed for anything else
631 * anymore, so we use it here to test the size
632 * of the Block Padding field.
633 */
634 while (s->block.compressed & 3) {
635 if (b->in_pos == b->in_size)
636 return XZ_OK;
637
638 if (b->in[b->in_pos++] != 0)
639 return XZ_DATA_ERROR;
640
641 ++s->block.compressed;
642 }
643
644 s->sequence = SEQ_BLOCK_CHECK;
645
646 case SEQ_BLOCK_CHECK:
647 if (s->check_type == XZ_CHECK_CRC32) {
648 ret = crc32_validate(s, b);
649 if (ret != XZ_STREAM_END)
650 return ret;
651 }
652#ifdef XZ_DEC_ANY_CHECK
653 else if (!check_skip(s, b)) {
654 return XZ_OK;
655 }
656#endif
657
658 s->sequence = SEQ_BLOCK_START;
659 break;
660
661 case SEQ_INDEX:
662 ret = dec_index(s, b);
663 if (ret != XZ_STREAM_END)
664 return ret;
665
666 s->sequence = SEQ_INDEX_PADDING;
667
668 case SEQ_INDEX_PADDING:
669 while ((s->index.size + (b->in_pos - s->in_start))
670 & 3) {
671 if (b->in_pos == b->in_size) {
672 index_update(s, b);
673 return XZ_OK;
674 }
675
676 if (b->in[b->in_pos++] != 0)
677 return XZ_DATA_ERROR;
678 }
679
680 /* Finish the CRC32 value and Index size. */
681 index_update(s, b);
682
683 /* Compare the hashes to validate the Index field. */
684 if (!memeq(&s->block.hash, &s->index.hash,
685 sizeof(s->block.hash)))
686 return XZ_DATA_ERROR;
687
688 s->sequence = SEQ_INDEX_CRC32;
689
690 case SEQ_INDEX_CRC32:
691 ret = crc32_validate(s, b);
692 if (ret != XZ_STREAM_END)
693 return ret;
694
695 s->temp.size = STREAM_HEADER_SIZE;
696 s->sequence = SEQ_STREAM_FOOTER;
697
698 case SEQ_STREAM_FOOTER:
699 if (!fill_temp(s, b))
700 return XZ_OK;
701
702 return dec_stream_footer(s);
703 }
704 }
705
706 /* Never reached */
707}
708
709/*
710 * xz_dec_run() is a wrapper for dec_main() to handle some special cases in
711 * multi-call and single-call decoding.
712 *
713 * In multi-call mode, we must return XZ_BUF_ERROR when it seems clear that we
714 * are not going to make any progress anymore. This is to prevent the caller
715 * from calling us infinitely when the input file is truncated or otherwise
716 * corrupt. Since zlib-style API allows that the caller fills the input buffer
717 * only when the decoder doesn't produce any new output, we have to be careful
718 * to avoid returning XZ_BUF_ERROR too easily: XZ_BUF_ERROR is returned only
719 * after the second consecutive call to xz_dec_run() that makes no progress.
720 *
721 * In single-call mode, if we couldn't decode everything and no error
722 * occurred, either the input is truncated or the output buffer is too small.
723 * Since we know that the last input byte never produces any output, we know
724 * that if all the input was consumed and decoding wasn't finished, the file
725 * must be corrupt. Otherwise the output buffer has to be too small or the
726 * file is corrupt in a way that decoding it produces too big output.
727 *
728 * If single-call decoding fails, we reset b->in_pos and b->out_pos back to
729 * their original values. This is because with some filter chains there won't
730 * be any valid uncompressed data in the output buffer unless the decoding
731 * actually succeeds (that's the price to pay of using the output buffer as
732 * the workspace).
733 */
734XZ_EXTERN enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b)
735{
736 size_t in_start;
737 size_t out_start;
738 enum xz_ret ret;
739
740 if (DEC_IS_SINGLE(s->mode))
741 xz_dec_reset(s);
742
743 in_start = b->in_pos;
744 out_start = b->out_pos;
745 ret = dec_main(s, b);
746
747 if (DEC_IS_SINGLE(s->mode)) {
748 if (ret == XZ_OK)
749 ret = b->in_pos == b->in_size
750 ? XZ_DATA_ERROR : XZ_BUF_ERROR;
751
752 if (ret != XZ_STREAM_END) {
753 b->in_pos = in_start;
754 b->out_pos = out_start;
755 }
756
757 } else if (ret == XZ_OK && in_start == b->in_pos
758 && out_start == b->out_pos) {
759 if (s->allow_buf_error)
760 ret = XZ_BUF_ERROR;
761
762 s->allow_buf_error = true;
763 } else {
764 s->allow_buf_error = false;
765 }
766
767 return ret;
768}
769
770XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max)
771{
772 struct xz_dec *s = kmalloc(sizeof(*s), GFP_KERNEL);
773 if (s == NULL)
774 return NULL;
775
776 s->mode = mode;
777
778#ifdef XZ_DEC_BCJ
779 s->bcj = xz_dec_bcj_create(DEC_IS_SINGLE(mode));
780 if (s->bcj == NULL)
781 goto error_bcj;
782#endif
783
784 s->lzma2 = xz_dec_lzma2_create(mode, dict_max);
785 if (s->lzma2 == NULL)
786 goto error_lzma2;
787
788 xz_dec_reset(s);
789 return s;
790
791error_lzma2:
792#ifdef XZ_DEC_BCJ
793 xz_dec_bcj_end(s->bcj);
794error_bcj:
795#endif
796 kfree(s);
797 return NULL;
798}
799
800XZ_EXTERN void xz_dec_reset(struct xz_dec *s)
801{
802 s->sequence = SEQ_STREAM_HEADER;
803 s->allow_buf_error = false;
804 s->pos = 0;
805 s->crc32 = 0;
806 memzero(&s->block, sizeof(s->block));
807 memzero(&s->index, sizeof(s->index));
808 s->temp.pos = 0;
809 s->temp.size = STREAM_HEADER_SIZE;
810}
811
812XZ_EXTERN void xz_dec_end(struct xz_dec *s)
813{
814 if (s != NULL) {
815 xz_dec_lzma2_end(s->lzma2);
816#ifdef XZ_DEC_BCJ
817 xz_dec_bcj_end(s->bcj);
818#endif
819 kfree(s);
820 }
821}
diff --git a/lib/xz/xz_dec_syms.c b/lib/xz/xz_dec_syms.c
new file mode 100644
index 000000000000..32eb3c03aede
--- /dev/null
+++ b/lib/xz/xz_dec_syms.c
@@ -0,0 +1,26 @@
1/*
2 * XZ decoder module information
3 *
4 * Author: Lasse Collin <lasse.collin@tukaani.org>
5 *
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
8 */
9
10#include <linux/module.h>
11#include <linux/xz.h>
12
13EXPORT_SYMBOL(xz_dec_init);
14EXPORT_SYMBOL(xz_dec_reset);
15EXPORT_SYMBOL(xz_dec_run);
16EXPORT_SYMBOL(xz_dec_end);
17
18MODULE_DESCRIPTION("XZ decompressor");
19MODULE_VERSION("1.0");
20MODULE_AUTHOR("Lasse Collin <lasse.collin@tukaani.org> and Igor Pavlov");
21
22/*
23 * This code is in the public domain, but in Linux it's simplest to just
24 * say it's GPL and consider the authors as the copyright holders.
25 */
26MODULE_LICENSE("GPL");
diff --git a/lib/xz/xz_dec_test.c b/lib/xz/xz_dec_test.c
new file mode 100644
index 000000000000..da28a19d6c98
--- /dev/null
+++ b/lib/xz/xz_dec_test.c
@@ -0,0 +1,220 @@
1/*
2 * XZ decoder tester
3 *
4 * Author: Lasse Collin <lasse.collin@tukaani.org>
5 *
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
8 */
9
10#include <linux/kernel.h>
11#include <linux/module.h>
12#include <linux/fs.h>
13#include <linux/uaccess.h>
14#include <linux/crc32.h>
15#include <linux/xz.h>
16
17/* Maximum supported dictionary size */
18#define DICT_MAX (1 << 20)
19
20/* Device name to pass to register_chrdev(). */
21#define DEVICE_NAME "xz_dec_test"
22
23/* Dynamically allocated device major number */
24static int device_major;
25
26/*
27 * We reuse the same decoder state, and thus can decode only one
28 * file at a time.
29 */
30static bool device_is_open;
31
32/* XZ decoder state */
33static struct xz_dec *state;
34
35/*
36 * Return value of xz_dec_run(). We need to avoid calling xz_dec_run() after
37 * it has returned XZ_STREAM_END, so we make this static.
38 */
39static enum xz_ret ret;
40
41/*
42 * Input and output buffers. The input buffer is used as a temporary safe
43 * place for the data coming from the userspace.
44 */
45static uint8_t buffer_in[1024];
46static uint8_t buffer_out[1024];
47
48/*
49 * Structure to pass the input and output buffers to the XZ decoder.
50 * A few of the fields are never modified so we initialize them here.
51 */
52static struct xz_buf buffers = {
53 .in = buffer_in,
54 .out = buffer_out,
55 .out_size = sizeof(buffer_out)
56};
57
58/*
59 * CRC32 of uncompressed data. This is used to give the user a simple way
60 * to check that the decoder produces correct output.
61 */
62static uint32_t crc;
63
64static int xz_dec_test_open(struct inode *i, struct file *f)
65{
66 if (device_is_open)
67 return -EBUSY;
68
69 device_is_open = true;
70
71 xz_dec_reset(state);
72 ret = XZ_OK;
73 crc = 0xFFFFFFFF;
74
75 buffers.in_pos = 0;
76 buffers.in_size = 0;
77 buffers.out_pos = 0;
78
79 printk(KERN_INFO DEVICE_NAME ": opened\n");
80 return 0;
81}
82
83static int xz_dec_test_release(struct inode *i, struct file *f)
84{
85 device_is_open = false;
86
87 if (ret == XZ_OK)
88 printk(KERN_INFO DEVICE_NAME ": input was truncated\n");
89
90 printk(KERN_INFO DEVICE_NAME ": closed\n");
91 return 0;
92}
93
94/*
95 * Decode the data given to us from the userspace. CRC32 of the uncompressed
96 * data is calculated and is printed at the end of successful decoding. The
97 * uncompressed data isn't stored anywhere for further use.
98 *
99 * The .xz file must have exactly one Stream and no Stream Padding. The data
100 * after the first Stream is considered to be garbage.
101 */
102static ssize_t xz_dec_test_write(struct file *file, const char __user *buf,
103 size_t size, loff_t *pos)
104{
105 size_t remaining;
106
107 if (ret != XZ_OK) {
108 if (size > 0)
109 printk(KERN_INFO DEVICE_NAME ": %zu bytes of "
110 "garbage at the end of the file\n",
111 size);
112
113 return -ENOSPC;
114 }
115
116 printk(KERN_INFO DEVICE_NAME ": decoding %zu bytes of input\n",
117 size);
118
119 remaining = size;
120 while ((remaining > 0 || buffers.out_pos == buffers.out_size)
121 && ret == XZ_OK) {
122 if (buffers.in_pos == buffers.in_size) {
123 buffers.in_pos = 0;
124 buffers.in_size = min(remaining, sizeof(buffer_in));
125 if (copy_from_user(buffer_in, buf, buffers.in_size))
126 return -EFAULT;
127
128 buf += buffers.in_size;
129 remaining -= buffers.in_size;
130 }
131
132 buffers.out_pos = 0;
133 ret = xz_dec_run(state, &buffers);
134 crc = crc32(crc, buffer_out, buffers.out_pos);
135 }
136
137 switch (ret) {
138 case XZ_OK:
139 printk(KERN_INFO DEVICE_NAME ": XZ_OK\n");
140 return size;
141
142 case XZ_STREAM_END:
143 printk(KERN_INFO DEVICE_NAME ": XZ_STREAM_END, "
144 "CRC32 = 0x%08X\n", ~crc);
145 return size - remaining - (buffers.in_size - buffers.in_pos);
146
147 case XZ_MEMLIMIT_ERROR:
148 printk(KERN_INFO DEVICE_NAME ": XZ_MEMLIMIT_ERROR\n");
149 break;
150
151 case XZ_FORMAT_ERROR:
152 printk(KERN_INFO DEVICE_NAME ": XZ_FORMAT_ERROR\n");
153 break;
154
155 case XZ_OPTIONS_ERROR:
156 printk(KERN_INFO DEVICE_NAME ": XZ_OPTIONS_ERROR\n");
157 break;
158
159 case XZ_DATA_ERROR:
160 printk(KERN_INFO DEVICE_NAME ": XZ_DATA_ERROR\n");
161 break;
162
163 case XZ_BUF_ERROR:
164 printk(KERN_INFO DEVICE_NAME ": XZ_BUF_ERROR\n");
165 break;
166
167 default:
168 printk(KERN_INFO DEVICE_NAME ": Bug detected!\n");
169 break;
170 }
171
172 return -EIO;
173}
174
175/* Allocate the XZ decoder state and register the character device. */
176static int __init xz_dec_test_init(void)
177{
178 static const struct file_operations fileops = {
179 .owner = THIS_MODULE,
180 .open = &xz_dec_test_open,
181 .release = &xz_dec_test_release,
182 .write = &xz_dec_test_write
183 };
184
185 state = xz_dec_init(XZ_PREALLOC, DICT_MAX);
186 if (state == NULL)
187 return -ENOMEM;
188
189 device_major = register_chrdev(0, DEVICE_NAME, &fileops);
190 if (device_major < 0) {
191 xz_dec_end(state);
192 return device_major;
193 }
194
195 printk(KERN_INFO DEVICE_NAME ": module loaded\n");
196 printk(KERN_INFO DEVICE_NAME ": Create a device node with "
197 "'mknod " DEVICE_NAME " c %d 0' and write .xz files "
198 "to it.\n", device_major);
199 return 0;
200}
201
202static void __exit xz_dec_test_exit(void)
203{
204 unregister_chrdev(device_major, DEVICE_NAME);
205 xz_dec_end(state);
206 printk(KERN_INFO DEVICE_NAME ": module unloaded\n");
207}
208
209module_init(xz_dec_test_init);
210module_exit(xz_dec_test_exit);
211
212MODULE_DESCRIPTION("XZ decompressor tester");
213MODULE_VERSION("1.0");
214MODULE_AUTHOR("Lasse Collin <lasse.collin@tukaani.org>");
215
216/*
217 * This code is in the public domain, but in Linux it's simplest to just
218 * say it's GPL and consider the authors as the copyright holders.
219 */
220MODULE_LICENSE("GPL");
diff --git a/lib/xz/xz_lzma2.h b/lib/xz/xz_lzma2.h
new file mode 100644
index 000000000000..071d67bee9f5
--- /dev/null
+++ b/lib/xz/xz_lzma2.h
@@ -0,0 +1,204 @@
1/*
2 * LZMA2 definitions
3 *
4 * Authors: Lasse Collin <lasse.collin@tukaani.org>
5 * Igor Pavlov <http://7-zip.org/>
6 *
7 * This file has been put into the public domain.
8 * You can do whatever you want with this file.
9 */
10
11#ifndef XZ_LZMA2_H
12#define XZ_LZMA2_H
13
14/* Range coder constants */
15#define RC_SHIFT_BITS 8
16#define RC_TOP_BITS 24
17#define RC_TOP_VALUE (1 << RC_TOP_BITS)
18#define RC_BIT_MODEL_TOTAL_BITS 11
19#define RC_BIT_MODEL_TOTAL (1 << RC_BIT_MODEL_TOTAL_BITS)
20#define RC_MOVE_BITS 5
21
22/*
23 * Maximum number of position states. A position state is the lowest pb
24 * number of bits of the current uncompressed offset. In some places there
25 * are different sets of probabilities for different position states.
26 */
27#define POS_STATES_MAX (1 << 4)
28
29/*
30 * This enum is used to track which LZMA symbols have occurred most recently
31 * and in which order. This information is used to predict the next symbol.
32 *
33 * Symbols:
34 * - Literal: One 8-bit byte
35 * - Match: Repeat a chunk of data at some distance
36 * - Long repeat: Multi-byte match at a recently seen distance
37 * - Short repeat: One-byte repeat at a recently seen distance
38 *
39 * The symbol names are in from STATE_oldest_older_previous. REP means
40 * either short or long repeated match, and NONLIT means any non-literal.
41 */
42enum lzma_state {
43 STATE_LIT_LIT,
44 STATE_MATCH_LIT_LIT,
45 STATE_REP_LIT_LIT,
46 STATE_SHORTREP_LIT_LIT,
47 STATE_MATCH_LIT,
48 STATE_REP_LIT,
49 STATE_SHORTREP_LIT,
50 STATE_LIT_MATCH,
51 STATE_LIT_LONGREP,
52 STATE_LIT_SHORTREP,
53 STATE_NONLIT_MATCH,
54 STATE_NONLIT_REP
55};
56
57/* Total number of states */
58#define STATES 12
59
60/* The lowest 7 states indicate that the previous state was a literal. */
61#define LIT_STATES 7
62
63/* Indicate that the latest symbol was a literal. */
64static inline void lzma_state_literal(enum lzma_state *state)
65{
66 if (*state <= STATE_SHORTREP_LIT_LIT)
67 *state = STATE_LIT_LIT;
68 else if (*state <= STATE_LIT_SHORTREP)
69 *state -= 3;
70 else
71 *state -= 6;
72}
73
74/* Indicate that the latest symbol was a match. */
75static inline void lzma_state_match(enum lzma_state *state)
76{
77 *state = *state < LIT_STATES ? STATE_LIT_MATCH : STATE_NONLIT_MATCH;
78}
79
80/* Indicate that the latest state was a long repeated match. */
81static inline void lzma_state_long_rep(enum lzma_state *state)
82{
83 *state = *state < LIT_STATES ? STATE_LIT_LONGREP : STATE_NONLIT_REP;
84}
85
86/* Indicate that the latest symbol was a short match. */
87static inline void lzma_state_short_rep(enum lzma_state *state)
88{
89 *state = *state < LIT_STATES ? STATE_LIT_SHORTREP : STATE_NONLIT_REP;
90}
91
92/* Test if the previous symbol was a literal. */
93static inline bool lzma_state_is_literal(enum lzma_state state)
94{
95 return state < LIT_STATES;
96}
97
98/* Each literal coder is divided in three sections:
99 * - 0x001-0x0FF: Without match byte
100 * - 0x101-0x1FF: With match byte; match bit is 0
101 * - 0x201-0x2FF: With match byte; match bit is 1
102 *
103 * Match byte is used when the previous LZMA symbol was something else than
104 * a literal (that is, it was some kind of match).
105 */
106#define LITERAL_CODER_SIZE 0x300
107
108/* Maximum number of literal coders */
109#define LITERAL_CODERS_MAX (1 << 4)
110
111/* Minimum length of a match is two bytes. */
112#define MATCH_LEN_MIN 2
113
114/* Match length is encoded with 4, 5, or 10 bits.
115 *
116 * Length Bits
117 * 2-9 4 = Choice=0 + 3 bits
118 * 10-17 5 = Choice=1 + Choice2=0 + 3 bits
119 * 18-273 10 = Choice=1 + Choice2=1 + 8 bits
120 */
121#define LEN_LOW_BITS 3
122#define LEN_LOW_SYMBOLS (1 << LEN_LOW_BITS)
123#define LEN_MID_BITS 3
124#define LEN_MID_SYMBOLS (1 << LEN_MID_BITS)
125#define LEN_HIGH_BITS 8
126#define LEN_HIGH_SYMBOLS (1 << LEN_HIGH_BITS)
127#define LEN_SYMBOLS (LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS + LEN_HIGH_SYMBOLS)
128
129/*
130 * Maximum length of a match is 273 which is a result of the encoding
131 * described above.
132 */
133#define MATCH_LEN_MAX (MATCH_LEN_MIN + LEN_SYMBOLS - 1)
134
135/*
136 * Different sets of probabilities are used for match distances that have
137 * very short match length: Lengths of 2, 3, and 4 bytes have a separate
138 * set of probabilities for each length. The matches with longer length
139 * use a shared set of probabilities.
140 */
141#define DIST_STATES 4
142
143/*
144 * Get the index of the appropriate probability array for decoding
145 * the distance slot.
146 */
147static inline uint32_t lzma_get_dist_state(uint32_t len)
148{
149 return len < DIST_STATES + MATCH_LEN_MIN
150 ? len - MATCH_LEN_MIN : DIST_STATES - 1;
151}
152
153/*
154 * The highest two bits of a 32-bit match distance are encoded using six bits.
155 * This six-bit value is called a distance slot. This way encoding a 32-bit
156 * value takes 6-36 bits, larger values taking more bits.
157 */
158#define DIST_SLOT_BITS 6
159#define DIST_SLOTS (1 << DIST_SLOT_BITS)
160
161/* Match distances up to 127 are fully encoded using probabilities. Since
162 * the highest two bits (distance slot) are always encoded using six bits,
163 * the distances 0-3 don't need any additional bits to encode, since the
164 * distance slot itself is the same as the actual distance. DIST_MODEL_START
165 * indicates the first distance slot where at least one additional bit is
166 * needed.
167 */
168#define DIST_MODEL_START 4
169
170/*
171 * Match distances greater than 127 are encoded in three pieces:
172 * - distance slot: the highest two bits
173 * - direct bits: 2-26 bits below the highest two bits
174 * - alignment bits: four lowest bits
175 *
176 * Direct bits don't use any probabilities.
177 *
178 * The distance slot value of 14 is for distances 128-191.
179 */
180#define DIST_MODEL_END 14
181
182/* Distance slots that indicate a distance <= 127. */
183#define FULL_DISTANCES_BITS (DIST_MODEL_END / 2)
184#define FULL_DISTANCES (1 << FULL_DISTANCES_BITS)
185
186/*
187 * For match distances greater than 127, only the highest two bits and the
188 * lowest four bits (alignment) is encoded using probabilities.
189 */
190#define ALIGN_BITS 4
191#define ALIGN_SIZE (1 << ALIGN_BITS)
192#define ALIGN_MASK (ALIGN_SIZE - 1)
193
194/* Total number of all probability variables */
195#define PROBS_TOTAL (1846 + LITERAL_CODERS_MAX * LITERAL_CODER_SIZE)
196
197/*
198 * LZMA remembers the four most recent match distances. Reusing these
199 * distances tends to take less space than re-encoding the actual
200 * distance value.
201 */
202#define REPS 4
203
204#endif
diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h
new file mode 100644
index 000000000000..a65633e06962
--- /dev/null
+++ b/lib/xz/xz_private.h
@@ -0,0 +1,156 @@
1/*
2 * Private includes and definitions
3 *
4 * Author: Lasse Collin <lasse.collin@tukaani.org>
5 *
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
8 */
9
10#ifndef XZ_PRIVATE_H
11#define XZ_PRIVATE_H
12
13#ifdef __KERNEL__
14# include <linux/xz.h>
15# include <asm/byteorder.h>
16# include <asm/unaligned.h>
17 /* XZ_PREBOOT may be defined only via decompress_unxz.c. */
18# ifndef XZ_PREBOOT
19# include <linux/slab.h>
20# include <linux/vmalloc.h>
21# include <linux/string.h>
22# ifdef CONFIG_XZ_DEC_X86
23# define XZ_DEC_X86
24# endif
25# ifdef CONFIG_XZ_DEC_POWERPC
26# define XZ_DEC_POWERPC
27# endif
28# ifdef CONFIG_XZ_DEC_IA64
29# define XZ_DEC_IA64
30# endif
31# ifdef CONFIG_XZ_DEC_ARM
32# define XZ_DEC_ARM
33# endif
34# ifdef CONFIG_XZ_DEC_ARMTHUMB
35# define XZ_DEC_ARMTHUMB
36# endif
37# ifdef CONFIG_XZ_DEC_SPARC
38# define XZ_DEC_SPARC
39# endif
40# define memeq(a, b, size) (memcmp(a, b, size) == 0)
41# define memzero(buf, size) memset(buf, 0, size)
42# endif
43# define get_le32(p) le32_to_cpup((const uint32_t *)(p))
44#else
45 /*
46 * For userspace builds, use a separate header to define the required
47 * macros and functions. This makes it easier to adapt the code into
48 * different environments and avoids clutter in the Linux kernel tree.
49 */
50# include "xz_config.h"
51#endif
52
53/* If no specific decoding mode is requested, enable support for all modes. */
54#if !defined(XZ_DEC_SINGLE) && !defined(XZ_DEC_PREALLOC) \
55 && !defined(XZ_DEC_DYNALLOC)
56# define XZ_DEC_SINGLE
57# define XZ_DEC_PREALLOC
58# define XZ_DEC_DYNALLOC
59#endif
60
61/*
62 * The DEC_IS_foo(mode) macros are used in "if" statements. If only some
63 * of the supported modes are enabled, these macros will evaluate to true or
64 * false at compile time and thus allow the compiler to omit unneeded code.
65 */
66#ifdef XZ_DEC_SINGLE
67# define DEC_IS_SINGLE(mode) ((mode) == XZ_SINGLE)
68#else
69# define DEC_IS_SINGLE(mode) (false)
70#endif
71
72#ifdef XZ_DEC_PREALLOC
73# define DEC_IS_PREALLOC(mode) ((mode) == XZ_PREALLOC)
74#else
75# define DEC_IS_PREALLOC(mode) (false)
76#endif
77
78#ifdef XZ_DEC_DYNALLOC
79# define DEC_IS_DYNALLOC(mode) ((mode) == XZ_DYNALLOC)
80#else
81# define DEC_IS_DYNALLOC(mode) (false)
82#endif
83
84#if !defined(XZ_DEC_SINGLE)
85# define DEC_IS_MULTI(mode) (true)
86#elif defined(XZ_DEC_PREALLOC) || defined(XZ_DEC_DYNALLOC)
87# define DEC_IS_MULTI(mode) ((mode) != XZ_SINGLE)
88#else
89# define DEC_IS_MULTI(mode) (false)
90#endif
91
92/*
93 * If any of the BCJ filter decoders are wanted, define XZ_DEC_BCJ.
94 * XZ_DEC_BCJ is used to enable generic support for BCJ decoders.
95 */
96#ifndef XZ_DEC_BCJ
97# if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \
98 || defined(XZ_DEC_IA64) || defined(XZ_DEC_ARM) \
99 || defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \
100 || defined(XZ_DEC_SPARC)
101# define XZ_DEC_BCJ
102# endif
103#endif
104
105/*
106 * Allocate memory for LZMA2 decoder. xz_dec_lzma2_reset() must be used
107 * before calling xz_dec_lzma2_run().
108 */
109XZ_EXTERN struct xz_dec_lzma2 *xz_dec_lzma2_create(enum xz_mode mode,
110 uint32_t dict_max);
111
112/*
113 * Decode the LZMA2 properties (one byte) and reset the decoder. Return
114 * XZ_OK on success, XZ_MEMLIMIT_ERROR if the preallocated dictionary is not
115 * big enough, and XZ_OPTIONS_ERROR if props indicates something that this
116 * decoder doesn't support.
117 */
118XZ_EXTERN enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s,
119 uint8_t props);
120
121/* Decode raw LZMA2 stream from b->in to b->out. */
122XZ_EXTERN enum xz_ret xz_dec_lzma2_run(struct xz_dec_lzma2 *s,
123 struct xz_buf *b);
124
125/* Free the memory allocated for the LZMA2 decoder. */
126XZ_EXTERN void xz_dec_lzma2_end(struct xz_dec_lzma2 *s);
127
128#ifdef XZ_DEC_BCJ
129/*
130 * Allocate memory for BCJ decoders. xz_dec_bcj_reset() must be used before
131 * calling xz_dec_bcj_run().
132 */
133XZ_EXTERN struct xz_dec_bcj *xz_dec_bcj_create(bool single_call);
134
135/*
136 * Decode the Filter ID of a BCJ filter. This implementation doesn't
137 * support custom start offsets, so no decoding of Filter Properties
138 * is needed. Returns XZ_OK if the given Filter ID is supported.
139 * Otherwise XZ_OPTIONS_ERROR is returned.
140 */
141XZ_EXTERN enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id);
142
143/*
144 * Decode raw BCJ + LZMA2 stream. This must be used only if there actually is
145 * a BCJ filter in the chain. If the chain has only LZMA2, xz_dec_lzma2_run()
146 * must be called directly.
147 */
148XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s,
149 struct xz_dec_lzma2 *lzma2,
150 struct xz_buf *b);
151
152/* Free the memory allocated for the BCJ filters. */
153#define xz_dec_bcj_end(s) kfree(s)
154#endif
155
156#endif
diff --git a/lib/xz/xz_stream.h b/lib/xz/xz_stream.h
new file mode 100644
index 000000000000..66cb5a7055ec
--- /dev/null
+++ b/lib/xz/xz_stream.h
@@ -0,0 +1,62 @@
1/*
2 * Definitions for handling the .xz file format
3 *
4 * Author: Lasse Collin <lasse.collin@tukaani.org>
5 *
6 * This file has been put into the public domain.
7 * You can do whatever you want with this file.
8 */
9
10#ifndef XZ_STREAM_H
11#define XZ_STREAM_H
12
13#if defined(__KERNEL__) && !XZ_INTERNAL_CRC32
14# include <linux/crc32.h>
15# undef crc32
16# define xz_crc32(buf, size, crc) \
17 (~crc32_le(~(uint32_t)(crc), buf, size))
18#endif
19
20/*
21 * See the .xz file format specification at
22 * http://tukaani.org/xz/xz-file-format.txt
23 * to understand the container format.
24 */
25
26#define STREAM_HEADER_SIZE 12
27
28#define HEADER_MAGIC "\3757zXZ"
29#define HEADER_MAGIC_SIZE 6
30
31#define FOOTER_MAGIC "YZ"
32#define FOOTER_MAGIC_SIZE 2
33
34/*
35 * Variable-length integer can hold a 63-bit unsigned integer or a special
36 * value indicating that the value is unknown.
37 *
38 * Experimental: vli_type can be defined to uint32_t to save a few bytes
39 * in code size (no effect on speed). Doing so limits the uncompressed and
40 * compressed size of the file to less than 256 MiB and may also weaken
41 * error detection slightly.
42 */
43typedef uint64_t vli_type;
44
45#define VLI_MAX ((vli_type)-1 / 2)
46#define VLI_UNKNOWN ((vli_type)-1)
47
48/* Maximum encoded size of a VLI */
49#define VLI_BYTES_MAX (sizeof(vli_type) * 8 / 7)
50
51/* Integrity Check types */
52enum xz_check {
53 XZ_CHECK_NONE = 0,
54 XZ_CHECK_CRC32 = 1,
55 XZ_CHECK_CRC64 = 4,
56 XZ_CHECK_SHA256 = 10
57};
58
59/* Maximum possible Check ID */
60#define XZ_CHECK_MAX 15
61
62#endif
diff --git a/lib/zlib_deflate/deflate.c b/lib/zlib_deflate/deflate.c
index 46a31e5f49c3..d63381e8e333 100644
--- a/lib/zlib_deflate/deflate.c
+++ b/lib/zlib_deflate/deflate.c
@@ -176,6 +176,7 @@ int zlib_deflateInit2(
176 deflate_state *s; 176 deflate_state *s;
177 int noheader = 0; 177 int noheader = 0;
178 deflate_workspace *mem; 178 deflate_workspace *mem;
179 char *next;
179 180
180 ush *overlay; 181 ush *overlay;
181 /* We overlay pending_buf and d_buf+l_buf. This works since the average 182 /* We overlay pending_buf and d_buf+l_buf. This works since the average
@@ -199,6 +200,21 @@ int zlib_deflateInit2(
199 strategy < 0 || strategy > Z_HUFFMAN_ONLY) { 200 strategy < 0 || strategy > Z_HUFFMAN_ONLY) {
200 return Z_STREAM_ERROR; 201 return Z_STREAM_ERROR;
201 } 202 }
203
204 /*
205 * Direct the workspace's pointers to the chunks that were allocated
206 * along with the deflate_workspace struct.
207 */
208 next = (char *) mem;
209 next += sizeof(*mem);
210 mem->window_memory = (Byte *) next;
211 next += zlib_deflate_window_memsize(windowBits);
212 mem->prev_memory = (Pos *) next;
213 next += zlib_deflate_prev_memsize(windowBits);
214 mem->head_memory = (Pos *) next;
215 next += zlib_deflate_head_memsize(memLevel);
216 mem->overlay_memory = next;
217
202 s = (deflate_state *) &(mem->deflate_memory); 218 s = (deflate_state *) &(mem->deflate_memory);
203 strm->state = (struct internal_state *)s; 219 strm->state = (struct internal_state *)s;
204 s->strm = strm; 220 s->strm = strm;
@@ -1247,7 +1263,18 @@ static block_state deflate_slow(
1247 return flush == Z_FINISH ? finish_done : block_done; 1263 return flush == Z_FINISH ? finish_done : block_done;
1248} 1264}
1249 1265
1250int zlib_deflate_workspacesize(void) 1266int zlib_deflate_workspacesize(int windowBits, int memLevel)
1251{ 1267{
1252 return sizeof(deflate_workspace); 1268 if (windowBits < 0) /* undocumented feature: suppress zlib header */
1269 windowBits = -windowBits;
1270
1271 /* Since the return value is typically passed to vmalloc() unchecked... */
1272 BUG_ON(memLevel < 1 || memLevel > MAX_MEM_LEVEL || windowBits < 9 ||
1273 windowBits > 15);
1274
1275 return sizeof(deflate_workspace)
1276 + zlib_deflate_window_memsize(windowBits)
1277 + zlib_deflate_prev_memsize(windowBits)
1278 + zlib_deflate_head_memsize(memLevel)
1279 + zlib_deflate_overlay_memsize(memLevel);
1253} 1280}
diff --git a/lib/zlib_deflate/defutil.h b/lib/zlib_deflate/defutil.h
index 6b15a909ca3f..b640b6402e99 100644
--- a/lib/zlib_deflate/defutil.h
+++ b/lib/zlib_deflate/defutil.h
@@ -241,12 +241,21 @@ typedef struct deflate_state {
241typedef struct deflate_workspace { 241typedef struct deflate_workspace {
242 /* State memory for the deflator */ 242 /* State memory for the deflator */
243 deflate_state deflate_memory; 243 deflate_state deflate_memory;
244 Byte window_memory[2 * (1 << MAX_WBITS)]; 244 Byte *window_memory;
245 Pos prev_memory[1 << MAX_WBITS]; 245 Pos *prev_memory;
246 Pos head_memory[1 << (MAX_MEM_LEVEL + 7)]; 246 Pos *head_memory;
247 char overlay_memory[(1 << (MAX_MEM_LEVEL + 6)) * (sizeof(ush)+2)]; 247 char *overlay_memory;
248} deflate_workspace; 248} deflate_workspace;
249 249
250#define zlib_deflate_window_memsize(windowBits) \
251 (2 * (1 << (windowBits)) * sizeof(Byte))
252#define zlib_deflate_prev_memsize(windowBits) \
253 ((1 << (windowBits)) * sizeof(Pos))
254#define zlib_deflate_head_memsize(memLevel) \
255 ((1 << ((memLevel)+7)) * sizeof(Pos))
256#define zlib_deflate_overlay_memsize(memLevel) \
257 ((1 << ((memLevel)+6)) * (sizeof(ush)+2))
258
250/* Output a byte on the stream. 259/* Output a byte on the stream.
251 * IN assertion: there is enough room in pending_buf. 260 * IN assertion: there is enough room in pending_buf.
252 */ 261 */
generated by cgit v1.2.2 (git 2.25.0) at 2025-07-07 15:33:50 -0400