diff options
author | Dave Jones <davej@redhat.com> | 2006-12-12 17:41:41 -0500 |
---|---|---|
committer | Dave Jones <davej@redhat.com> | 2006-12-12 17:41:41 -0500 |
commit | c4366889dda8110247be59ca41fddb82951a8c26 (patch) | |
tree | 705c1a996bed8fd48ce94ff33ec9fd00f9b94875 /Documentation/fault-injection | |
parent | db2fb9db5735cc532fd4fc55e94b9a3c3750378e (diff) | |
parent | e1036502e5263851259d147771226161e5ccc85a (diff) |
Merge ../linus
Conflicts:
drivers/cpufreq/cpufreq.c
Diffstat (limited to 'Documentation/fault-injection')
-rw-r--r-- | Documentation/fault-injection/failcmd.sh | 4 | ||||
-rw-r--r-- | Documentation/fault-injection/failmodule.sh | 31 | ||||
-rw-r--r-- | Documentation/fault-injection/fault-injection.txt | 225 |
3 files changed, 260 insertions, 0 deletions
diff --git a/Documentation/fault-injection/failcmd.sh b/Documentation/fault-injection/failcmd.sh new file mode 100644 index 000000000000..63177aba8106 --- /dev/null +++ b/Documentation/fault-injection/failcmd.sh | |||
@@ -0,0 +1,4 @@ | |||
1 | #!/bin/bash | ||
2 | |||
3 | echo 1 > /proc/self/make-it-fail | ||
4 | exec $* | ||
diff --git a/Documentation/fault-injection/failmodule.sh b/Documentation/fault-injection/failmodule.sh new file mode 100644 index 000000000000..474a8b971f9c --- /dev/null +++ b/Documentation/fault-injection/failmodule.sh | |||
@@ -0,0 +1,31 @@ | |||
1 | #!/bin/bash | ||
2 | # | ||
3 | # Usage: failmodule <failname> <modulename> [stacktrace-depth] | ||
4 | # | ||
5 | # <failname>: "failslab", "fail_alloc_page", or "fail_make_request" | ||
6 | # | ||
7 | # <modulename>: module name that you want to inject faults. | ||
8 | # | ||
9 | # [stacktrace-depth]: the maximum number of stacktrace walking allowed | ||
10 | # | ||
11 | |||
12 | STACKTRACE_DEPTH=5 | ||
13 | if [ $# -gt 2 ]; then | ||
14 | STACKTRACE_DEPTH=$3 | ||
15 | fi | ||
16 | |||
17 | if [ ! -d /debug/$1 ]; then | ||
18 | echo "Fault-injection $1 does not exist" >&2 | ||
19 | exit 1 | ||
20 | fi | ||
21 | if [ ! -d /sys/module/$2 ]; then | ||
22 | echo "Module $2 does not exist" >&2 | ||
23 | exit 1 | ||
24 | fi | ||
25 | |||
26 | # Disable any fault injection | ||
27 | echo 0 > /debug/$1/stacktrace-depth | ||
28 | |||
29 | echo `cat /sys/module/$2/sections/.text` > /debug/$1/require-start | ||
30 | echo `cat /sys/module/$2/sections/.exit.text` > /debug/$1/require-end | ||
31 | echo $STACKTRACE_DEPTH > /debug/$1/stacktrace-depth | ||
diff --git a/Documentation/fault-injection/fault-injection.txt b/Documentation/fault-injection/fault-injection.txt new file mode 100644 index 000000000000..b7ca560b9340 --- /dev/null +++ b/Documentation/fault-injection/fault-injection.txt | |||
@@ -0,0 +1,225 @@ | |||
1 | Fault injection capabilities infrastructure | ||
2 | =========================================== | ||
3 | |||
4 | See also drivers/md/faulty.c and "every_nth" module option for scsi_debug. | ||
5 | |||
6 | |||
7 | Available fault injection capabilities | ||
8 | -------------------------------------- | ||
9 | |||
10 | o failslab | ||
11 | |||
12 | injects slab allocation failures. (kmalloc(), kmem_cache_alloc(), ...) | ||
13 | |||
14 | o fail_page_alloc | ||
15 | |||
16 | injects page allocation failures. (alloc_pages(), get_free_pages(), ...) | ||
17 | |||
18 | o fail_make_request | ||
19 | |||
20 | injects disk IO errors on devices permitted by setting | ||
21 | /sys/block/<device>/make-it-fail or | ||
22 | /sys/block/<device>/<partition>/make-it-fail. (generic_make_request()) | ||
23 | |||
24 | Configure fault-injection capabilities behavior | ||
25 | ----------------------------------------------- | ||
26 | |||
27 | o debugfs entries | ||
28 | |||
29 | fault-inject-debugfs kernel module provides some debugfs entries for runtime | ||
30 | configuration of fault-injection capabilities. | ||
31 | |||
32 | - /debug/fail*/probability: | ||
33 | |||
34 | likelihood of failure injection, in percent. | ||
35 | Format: <percent> | ||
36 | |||
37 | Note that one-failure-per-hundred is a very high error rate | ||
38 | for some testcases. Consider setting probability=100 and configure | ||
39 | /debug/fail*/interval for such testcases. | ||
40 | |||
41 | - /debug/fail*/interval: | ||
42 | |||
43 | specifies the interval between failures, for calls to | ||
44 | should_fail() that pass all the other tests. | ||
45 | |||
46 | Note that if you enable this, by setting interval>1, you will | ||
47 | probably want to set probability=100. | ||
48 | |||
49 | - /debug/fail*/times: | ||
50 | |||
51 | specifies how many times failures may happen at most. | ||
52 | A value of -1 means "no limit". | ||
53 | |||
54 | - /debug/fail*/space: | ||
55 | |||
56 | specifies an initial resource "budget", decremented by "size" | ||
57 | on each call to should_fail(,size). Failure injection is | ||
58 | suppressed until "space" reaches zero. | ||
59 | |||
60 | - /debug/fail*/verbose | ||
61 | |||
62 | Format: { 0 | 1 | 2 } | ||
63 | specifies the verbosity of the messages when failure is | ||
64 | injected. '0' means no messages; '1' will print only a single | ||
65 | log line per failure; '2' will print a call trace too -- useful | ||
66 | to debug the problems revealed by fault injection. | ||
67 | |||
68 | - /debug/fail*/task-filter: | ||
69 | |||
70 | Format: { 'Y' | 'N' } | ||
71 | A value of 'N' disables filtering by process (default). | ||
72 | Any positive value limits failures to only processes indicated by | ||
73 | /proc/<pid>/make-it-fail==1. | ||
74 | |||
75 | - /debug/fail*/require-start: | ||
76 | - /debug/fail*/require-end: | ||
77 | - /debug/fail*/reject-start: | ||
78 | - /debug/fail*/reject-end: | ||
79 | |||
80 | specifies the range of virtual addresses tested during | ||
81 | stacktrace walking. Failure is injected only if some caller | ||
82 | in the walked stacktrace lies within the required range, and | ||
83 | none lies within the rejected range. | ||
84 | Default required range is [0,ULONG_MAX) (whole of virtual address space). | ||
85 | Default rejected range is [0,0). | ||
86 | |||
87 | - /debug/fail*/stacktrace-depth: | ||
88 | |||
89 | specifies the maximum stacktrace depth walked during search | ||
90 | for a caller within [require-start,require-end) OR | ||
91 | [reject-start,reject-end). | ||
92 | |||
93 | - /debug/fail_page_alloc/ignore-gfp-highmem: | ||
94 | |||
95 | Format: { 'Y' | 'N' } | ||
96 | default is 'N', setting it to 'Y' won't inject failures into | ||
97 | highmem/user allocations. | ||
98 | |||
99 | - /debug/failslab/ignore-gfp-wait: | ||
100 | - /debug/fail_page_alloc/ignore-gfp-wait: | ||
101 | |||
102 | Format: { 'Y' | 'N' } | ||
103 | default is 'N', setting it to 'Y' will inject failures | ||
104 | only into non-sleep allocations (GFP_ATOMIC allocations). | ||
105 | |||
106 | o Boot option | ||
107 | |||
108 | In order to inject faults while debugfs is not available (early boot time), | ||
109 | use the boot option: | ||
110 | |||
111 | failslab= | ||
112 | fail_page_alloc= | ||
113 | fail_make_request=<interval>,<probability>,<space>,<times> | ||
114 | |||
115 | How to add new fault injection capability | ||
116 | ----------------------------------------- | ||
117 | |||
118 | o #include <linux/fault-inject.h> | ||
119 | |||
120 | o define the fault attributes | ||
121 | |||
122 | DECLARE_FAULT_INJECTION(name); | ||
123 | |||
124 | Please see the definition of struct fault_attr in fault-inject.h | ||
125 | for details. | ||
126 | |||
127 | o provide a way to configure fault attributes | ||
128 | |||
129 | - boot option | ||
130 | |||
131 | If you need to enable the fault injection capability from boot time, you can | ||
132 | provide boot option to configure it. There is a helper function for it: | ||
133 | |||
134 | setup_fault_attr(attr, str); | ||
135 | |||
136 | - debugfs entries | ||
137 | |||
138 | failslab, fail_page_alloc, and fail_make_request use this way. | ||
139 | Helper functions: | ||
140 | |||
141 | init_fault_attr_entries(entries, attr, name); | ||
142 | void cleanup_fault_attr_entries(entries); | ||
143 | |||
144 | - module parameters | ||
145 | |||
146 | If the scope of the fault injection capability is limited to a | ||
147 | single kernel module, it is better to provide module parameters to | ||
148 | configure the fault attributes. | ||
149 | |||
150 | o add a hook to insert failures | ||
151 | |||
152 | Upon should_fail() returning true, client code should inject a failure. | ||
153 | |||
154 | should_fail(attr, size); | ||
155 | |||
156 | Application Examples | ||
157 | -------------------- | ||
158 | |||
159 | o inject slab allocation failures into module init/cleanup code | ||
160 | |||
161 | ------------------------------------------------------------------------------ | ||
162 | #!/bin/bash | ||
163 | |||
164 | FAILCMD=Documentation/fault-injection/failcmd.sh | ||
165 | BLACKLIST="root_plug evbug" | ||
166 | |||
167 | FAILNAME=failslab | ||
168 | echo Y > /debug/$FAILNAME/task-filter | ||
169 | echo 10 > /debug/$FAILNAME/probability | ||
170 | echo 100 > /debug/$FAILNAME/interval | ||
171 | echo -1 > /debug/$FAILNAME/times | ||
172 | echo 2 > /debug/$FAILNAME/verbose | ||
173 | echo 1 > /debug/$FAILNAME/ignore-gfp-wait | ||
174 | |||
175 | blacklist() | ||
176 | { | ||
177 | echo $BLACKLIST | grep $1 > /dev/null 2>&1 | ||
178 | } | ||
179 | |||
180 | oops() | ||
181 | { | ||
182 | dmesg | grep BUG > /dev/null 2>&1 | ||
183 | } | ||
184 | |||
185 | find /lib/modules/`uname -r` -name '*.ko' -exec basename {} .ko \; | | ||
186 | while read i | ||
187 | do | ||
188 | oops && exit 1 | ||
189 | |||
190 | if ! blacklist $i | ||
191 | then | ||
192 | echo inserting $i... | ||
193 | bash $FAILCMD modprobe $i | ||
194 | fi | ||
195 | done | ||
196 | |||
197 | lsmod | awk '{ if ($3 == 0) { print $1 } }' | | ||
198 | while read i | ||
199 | do | ||
200 | oops && exit 1 | ||
201 | |||
202 | if ! blacklist $i | ||
203 | then | ||
204 | echo removing $i... | ||
205 | bash $FAILCMD modprobe -r $i | ||
206 | fi | ||
207 | done | ||
208 | |||
209 | ------------------------------------------------------------------------------ | ||
210 | |||
211 | o inject slab allocation failures only for a specific module | ||
212 | |||
213 | ------------------------------------------------------------------------------ | ||
214 | #!/bin/bash | ||
215 | |||
216 | FAILMOD=Documentation/fault-injection/failmodule.sh | ||
217 | |||
218 | echo injecting errors into the module $1... | ||
219 | |||
220 | modprobe $1 | ||
221 | bash $FAILMOD failslab $1 10 | ||
222 | echo 25 > /debug/failslab/probability | ||
223 | |||
224 | ------------------------------------------------------------------------------ | ||
225 | |||