aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/fault-injection/provoke-crashes.txt38
-rw-r--r--drivers/misc/lkdtm.c472
-rw-r--r--lib/Kconfig.debug5
3 files changed, 430 insertions, 85 deletions
diff --git a/Documentation/fault-injection/provoke-crashes.txt b/Documentation/fault-injection/provoke-crashes.txt
new file mode 100644
index 000000000000..7a9d3d81525b
--- /dev/null
+++ b/Documentation/fault-injection/provoke-crashes.txt
@@ -0,0 +1,38 @@
1The lkdtm module provides an interface to crash or injure the kernel at
2predefined crashpoints to evaluate the reliability of crash dumps obtained
3using different dumping solutions. The module uses KPROBEs to instrument
4crashing points, but can also crash the kernel directly without KRPOBE
5support.
6
7
8You can provide the way either through module arguments when inserting
9the module, or through a debugfs interface.
10
11Usage: insmod lkdtm.ko [recur_count={>0}] cpoint_name=<> cpoint_type=<>
12 [cpoint_count={>0}]
13
14 recur_count : Recursion level for the stack overflow test. Default is 10.
15
16 cpoint_name : Crash point where the kernel is to be crashed. It can be
17 one of INT_HARDWARE_ENTRY, INT_HW_IRQ_EN, INT_TASKLET_ENTRY,
18 FS_DEVRW, MEM_SWAPOUT, TIMERADD, SCSI_DISPATCH_CMD,
19 IDE_CORE_CP, DIRECT
20
21 cpoint_type : Indicates the action to be taken on hitting the crash point.
22 It can be one of PANIC, BUG, EXCEPTION, LOOP, OVERFLOW,
23 CORRUPT_STACK, UNALIGNED_LOAD_STORE_WRITE, OVERWRITE_ALLOCATION,
24 WRITE_AFTER_FREE,
25
26 cpoint_count : Indicates the number of times the crash point is to be hit
27 to trigger an action. The default is 10.
28
29You can also induce failures by mounting debugfs and writing the type to
30<mountpoint>/provoke-crash/<crashpoint>. E.g.,
31
32 mount -t debugfs debugfs /mnt
33 echo EXCEPTION > /mnt/provoke-crash/INT_HARDWARE_ENTRY
34
35
36A special file is `DIRECT' which will induce the crash directly without
37KPROBE instrumentation. This mode is the only one available when the module
38is built on a kernel without KPROBEs support.
diff --git a/drivers/misc/lkdtm.c b/drivers/misc/lkdtm.c
index 3648b23d5c92..4a0648301fdf 100644
--- a/drivers/misc/lkdtm.c
+++ b/drivers/misc/lkdtm.c
@@ -26,21 +26,9 @@
26 * It is adapted from the Linux Kernel Dump Test Tool by 26 * It is adapted from the Linux Kernel Dump Test Tool by
27 * Fernando Luis Vazquez Cao <http://lkdtt.sourceforge.net> 27 * Fernando Luis Vazquez Cao <http://lkdtt.sourceforge.net>
28 * 28 *
29 * Usage : insmod lkdtm.ko [recur_count={>0}] cpoint_name=<> cpoint_type=<> 29 * Debugfs support added by Simon Kagstrom <simon.kagstrom@netinsight.net>
30 * [cpoint_count={>0}]
31 * 30 *
32 * recur_count : Recursion level for the stack overflow test. Default is 10. 31 * See Documentation/fault-injection/provoke-crashes.txt for instructions
33 *
34 * cpoint_name : Crash point where the kernel is to be crashed. It can be
35 * one of INT_HARDWARE_ENTRY, INT_HW_IRQ_EN, INT_TASKLET_ENTRY,
36 * FS_DEVRW, MEM_SWAPOUT, TIMERADD, SCSI_DISPATCH_CMD,
37 * IDE_CORE_CP
38 *
39 * cpoint_type : Indicates the action to be taken on hitting the crash point.
40 * It can be one of PANIC, BUG, EXCEPTION, LOOP, OVERFLOW
41 *
42 * cpoint_count : Indicates the number of times the crash point is to be hit
43 * to trigger an action. The default is 10.
44 */ 32 */
45 33
46#include <linux/kernel.h> 34#include <linux/kernel.h>
@@ -53,13 +41,12 @@
53#include <linux/interrupt.h> 41#include <linux/interrupt.h>
54#include <linux/hrtimer.h> 42#include <linux/hrtimer.h>
55#include <scsi/scsi_cmnd.h> 43#include <scsi/scsi_cmnd.h>
44#include <linux/debugfs.h>
56 45
57#ifdef CONFIG_IDE 46#ifdef CONFIG_IDE
58#include <linux/ide.h> 47#include <linux/ide.h>
59#endif 48#endif
60 49
61#define NUM_CPOINTS 8
62#define NUM_CPOINT_TYPES 5
63#define DEFAULT_COUNT 10 50#define DEFAULT_COUNT 10
64#define REC_NUM_DEFAULT 10 51#define REC_NUM_DEFAULT 10
65 52
@@ -72,7 +59,8 @@ enum cname {
72 MEM_SWAPOUT, 59 MEM_SWAPOUT,
73 TIMERADD, 60 TIMERADD,
74 SCSI_DISPATCH_CMD, 61 SCSI_DISPATCH_CMD,
75 IDE_CORE_CP 62 IDE_CORE_CP,
63 DIRECT,
76}; 64};
77 65
78enum ctype { 66enum ctype {
@@ -81,7 +69,11 @@ enum ctype {
81 BUG, 69 BUG,
82 EXCEPTION, 70 EXCEPTION,
83 LOOP, 71 LOOP,
84 OVERFLOW 72 OVERFLOW,
73 CORRUPT_STACK,
74 UNALIGNED_LOAD_STORE_WRITE,
75 OVERWRITE_ALLOCATION,
76 WRITE_AFTER_FREE,
85}; 77};
86 78
87static char* cp_name[] = { 79static char* cp_name[] = {
@@ -92,7 +84,8 @@ static char* cp_name[] = {
92 "MEM_SWAPOUT", 84 "MEM_SWAPOUT",
93 "TIMERADD", 85 "TIMERADD",
94 "SCSI_DISPATCH_CMD", 86 "SCSI_DISPATCH_CMD",
95 "IDE_CORE_CP" 87 "IDE_CORE_CP",
88 "DIRECT",
96}; 89};
97 90
98static char* cp_type[] = { 91static char* cp_type[] = {
@@ -100,7 +93,11 @@ static char* cp_type[] = {
100 "BUG", 93 "BUG",
101 "EXCEPTION", 94 "EXCEPTION",
102 "LOOP", 95 "LOOP",
103 "OVERFLOW" 96 "OVERFLOW",
97 "CORRUPT_STACK",
98 "UNALIGNED_LOAD_STORE_WRITE",
99 "OVERWRITE_ALLOCATION",
100 "WRITE_AFTER_FREE",
104}; 101};
105 102
106static struct jprobe lkdtm; 103static struct jprobe lkdtm;
@@ -193,34 +190,66 @@ int jp_generic_ide_ioctl(ide_drive_t *drive, struct file *file,
193} 190}
194#endif 191#endif
195 192
193/* Return the crashpoint number or NONE if the name is invalid */
194static enum ctype parse_cp_type(const char *what, size_t count)
195{
196 int i;
197
198 for (i = 0; i < ARRAY_SIZE(cp_type); i++) {
199 if (!strcmp(what, cp_type[i]))
200 return i + 1;
201 }
202
203 return NONE;
204}
205
206static const char *cp_type_to_str(enum ctype type)
207{
208 if (type == NONE || type < 0 || type > ARRAY_SIZE(cp_type))
209 return "None";
210
211 return cp_type[type - 1];
212}
213
214static const char *cp_name_to_str(enum cname name)
215{
216 if (name == INVALID || name < 0 || name > ARRAY_SIZE(cp_name))
217 return "INVALID";
218
219 return cp_name[name - 1];
220}
221
222
196static int lkdtm_parse_commandline(void) 223static int lkdtm_parse_commandline(void)
197{ 224{
198 int i; 225 int i;
199 226
200 if (cpoint_name == NULL || cpoint_type == NULL || 227 if (cpoint_count < 1 || recur_count < 1)
201 cpoint_count < 1 || recur_count < 1)
202 return -EINVAL; 228 return -EINVAL;
203 229
204 for (i = 0; i < NUM_CPOINTS; ++i) { 230 count = cpoint_count;
231
232 /* No special parameters */
233 if (!cpoint_type && !cpoint_name)
234 return 0;
235
236 /* Neither or both of these need to be set */
237 if (!cpoint_type || !cpoint_name)
238 return -EINVAL;
239
240 cptype = parse_cp_type(cpoint_type, strlen(cpoint_type));
241 if (cptype == NONE)
242 return -EINVAL;
243
244 for (i = 0; i < ARRAY_SIZE(cp_name); i++) {
205 if (!strcmp(cpoint_name, cp_name[i])) { 245 if (!strcmp(cpoint_name, cp_name[i])) {
206 cpoint = i + 1; 246 cpoint = i + 1;
207 break; 247 return 0;
208 }
209 }
210
211 for (i = 0; i < NUM_CPOINT_TYPES; ++i) {
212 if (!strcmp(cpoint_type, cp_type[i])) {
213 cptype = i + 1;
214 break;
215 } 248 }
216 } 249 }
217 250
218 if (cpoint == INVALID || cptype == NONE) 251 /* Could not find a valid crash point */
219 return -EINVAL; 252 return -EINVAL;
220
221 count = cpoint_count;
222
223 return 0;
224} 253}
225 254
226static int recursive_loop(int a) 255static int recursive_loop(int a)
@@ -235,53 +264,92 @@ static int recursive_loop(int a)
235 return recursive_loop(a); 264 return recursive_loop(a);
236} 265}
237 266
238void lkdtm_handler(void) 267static void lkdtm_do_action(enum ctype which)
239{ 268{
240 printk(KERN_INFO "lkdtm : Crash point %s of type %s hit\n", 269 switch (which) {
241 cpoint_name, cpoint_type); 270 case PANIC:
242 --count; 271 panic("dumptest");
272 break;
273 case BUG:
274 BUG();
275 break;
276 case EXCEPTION:
277 *((int *) 0) = 0;
278 break;
279 case LOOP:
280 for (;;)
281 ;
282 break;
283 case OVERFLOW:
284 (void) recursive_loop(0);
285 break;
286 case CORRUPT_STACK: {
287 volatile u32 data[8];
288 volatile u32 *p = data;
289
290 p[12] = 0x12345678;
291 break;
292 }
293 case UNALIGNED_LOAD_STORE_WRITE: {
294 static u8 data[5] __attribute__((aligned(4))) = {1, 2,
295 3, 4, 5};
296 u32 *p;
297 u32 val = 0x12345678;
298
299 p = (u32 *)(data + 1);
300 if (*p == 0)
301 val = 0x87654321;
302 *p = val;
303 break;
304 }
305 case OVERWRITE_ALLOCATION: {
306 size_t len = 1020;
307 u32 *data = kmalloc(len, GFP_KERNEL);
308
309 data[1024 / sizeof(u32)] = 0x12345678;
310 kfree(data);
311 break;
312 }
313 case WRITE_AFTER_FREE: {
314 size_t len = 1024;
315 u32 *data = kmalloc(len, GFP_KERNEL);
316
317 kfree(data);
318 schedule();
319 memset(data, 0x78, len);
320 break;
321 }
322 case NONE:
323 default:
324 break;
325 }
326
327}
328
329static void lkdtm_handler(void)
330{
331 count--;
332 printk(KERN_INFO "lkdtm: Crash point %s of type %s hit, trigger in %d rounds\n",
333 cp_name_to_str(cpoint), cp_type_to_str(cptype), count);
243 334
244 if (count == 0) { 335 if (count == 0) {
245 switch (cptype) { 336 lkdtm_do_action(cptype);
246 case NONE:
247 break;
248 case PANIC:
249 printk(KERN_INFO "lkdtm : PANIC\n");
250 panic("dumptest");
251 break;
252 case BUG:
253 printk(KERN_INFO "lkdtm : BUG\n");
254 BUG();
255 break;
256 case EXCEPTION:
257 printk(KERN_INFO "lkdtm : EXCEPTION\n");
258 *((int *) 0) = 0;
259 break;
260 case LOOP:
261 printk(KERN_INFO "lkdtm : LOOP\n");
262 for (;;);
263 break;
264 case OVERFLOW:
265 printk(KERN_INFO "lkdtm : OVERFLOW\n");
266 (void) recursive_loop(0);
267 break;
268 default:
269 break;
270 }
271 count = cpoint_count; 337 count = cpoint_count;
272 } 338 }
273} 339}
274 340
275static int __init lkdtm_module_init(void) 341static int lkdtm_register_cpoint(enum cname which)
276{ 342{
277 int ret; 343 int ret;
278 344
279 if (lkdtm_parse_commandline() == -EINVAL) { 345 cpoint = INVALID;
280 printk(KERN_INFO "lkdtm : Invalid command\n"); 346 if (lkdtm.entry != NULL)
281 return -EINVAL; 347 unregister_jprobe(&lkdtm);
282 }
283 348
284 switch (cpoint) { 349 switch (which) {
350 case DIRECT:
351 lkdtm_do_action(cptype);
352 return 0;
285 case INT_HARDWARE_ENTRY: 353 case INT_HARDWARE_ENTRY:
286 lkdtm.kp.symbol_name = "do_IRQ"; 354 lkdtm.kp.symbol_name = "do_IRQ";
287 lkdtm.entry = (kprobe_opcode_t*) jp_do_irq; 355 lkdtm.entry = (kprobe_opcode_t*) jp_do_irq;
@@ -315,28 +383,268 @@ static int __init lkdtm_module_init(void)
315 lkdtm.kp.symbol_name = "generic_ide_ioctl"; 383 lkdtm.kp.symbol_name = "generic_ide_ioctl";
316 lkdtm.entry = (kprobe_opcode_t*) jp_generic_ide_ioctl; 384 lkdtm.entry = (kprobe_opcode_t*) jp_generic_ide_ioctl;
317#else 385#else
318 printk(KERN_INFO "lkdtm : Crash point not available\n"); 386 printk(KERN_INFO "lkdtm: Crash point not available\n");
387 return -EINVAL;
319#endif 388#endif
320 break; 389 break;
321 default: 390 default:
322 printk(KERN_INFO "lkdtm : Invalid Crash Point\n"); 391 printk(KERN_INFO "lkdtm: Invalid Crash Point\n");
323 break; 392 return -EINVAL;
324 } 393 }
325 394
395 cpoint = which;
326 if ((ret = register_jprobe(&lkdtm)) < 0) { 396 if ((ret = register_jprobe(&lkdtm)) < 0) {
327 printk(KERN_INFO "lkdtm : Couldn't register jprobe\n"); 397 printk(KERN_INFO "lkdtm: Couldn't register jprobe\n");
328 return ret; 398 cpoint = INVALID;
399 }
400
401 return ret;
402}
403
404static ssize_t do_register_entry(enum cname which, struct file *f,
405 const char __user *user_buf, size_t count, loff_t *off)
406{
407 char *buf;
408 int err;
409
410 if (count >= PAGE_SIZE)
411 return -EINVAL;
412
413 buf = (char *)__get_free_page(GFP_KERNEL);
414 if (!buf)
415 return -ENOMEM;
416 if (copy_from_user(buf, user_buf, count)) {
417 free_page((unsigned long) buf);
418 return -EFAULT;
419 }
420 /* NULL-terminate and remove enter */
421 buf[count] = '\0';
422 strim(buf);
423
424 cptype = parse_cp_type(buf, count);
425 free_page((unsigned long) buf);
426
427 if (cptype == NONE)
428 return -EINVAL;
429
430 err = lkdtm_register_cpoint(which);
431 if (err < 0)
432 return err;
433
434 *off += count;
435
436 return count;
437}
438
439/* Generic read callback that just prints out the available crash types */
440static ssize_t lkdtm_debugfs_read(struct file *f, char __user *user_buf,
441 size_t count, loff_t *off)
442{
443 char *buf;
444 int i, n, out;
445
446 buf = (char *)__get_free_page(GFP_KERNEL);
447
448 n = snprintf(buf, PAGE_SIZE, "Available crash types:\n");
449 for (i = 0; i < ARRAY_SIZE(cp_type); i++)
450 n += snprintf(buf + n, PAGE_SIZE - n, "%s\n", cp_type[i]);
451 buf[n] = '\0';
452
453 out = simple_read_from_buffer(user_buf, count, off,
454 buf, n);
455 free_page((unsigned long) buf);
456
457 return out;
458}
459
460static int lkdtm_debugfs_open(struct inode *inode, struct file *file)
461{
462 return 0;
463}
464
465
466static ssize_t int_hardware_entry(struct file *f, const char __user *buf,
467 size_t count, loff_t *off)
468{
469 return do_register_entry(INT_HARDWARE_ENTRY, f, buf, count, off);
470}
471
472static ssize_t int_hw_irq_en(struct file *f, const char __user *buf,
473 size_t count, loff_t *off)
474{
475 return do_register_entry(INT_HW_IRQ_EN, f, buf, count, off);
476}
477
478static ssize_t int_tasklet_entry(struct file *f, const char __user *buf,
479 size_t count, loff_t *off)
480{
481 return do_register_entry(INT_TASKLET_ENTRY, f, buf, count, off);
482}
483
484static ssize_t fs_devrw_entry(struct file *f, const char __user *buf,
485 size_t count, loff_t *off)
486{
487 return do_register_entry(FS_DEVRW, f, buf, count, off);
488}
489
490static ssize_t mem_swapout_entry(struct file *f, const char __user *buf,
491 size_t count, loff_t *off)
492{
493 return do_register_entry(MEM_SWAPOUT, f, buf, count, off);
494}
495
496static ssize_t timeradd_entry(struct file *f, const char __user *buf,
497 size_t count, loff_t *off)
498{
499 return do_register_entry(TIMERADD, f, buf, count, off);
500}
501
502static ssize_t scsi_dispatch_cmd_entry(struct file *f,
503 const char __user *buf, size_t count, loff_t *off)
504{
505 return do_register_entry(SCSI_DISPATCH_CMD, f, buf, count, off);
506}
507
508static ssize_t ide_core_cp_entry(struct file *f, const char __user *buf,
509 size_t count, loff_t *off)
510{
511 return do_register_entry(IDE_CORE_CP, f, buf, count, off);
512}
513
514/* Special entry to just crash directly. Available without KPROBEs */
515static ssize_t direct_entry(struct file *f, const char __user *user_buf,
516 size_t count, loff_t *off)
517{
518 enum ctype type;
519 char *buf;
520
521 if (count >= PAGE_SIZE)
522 return -EINVAL;
523 if (count < 1)
524 return -EINVAL;
525
526 buf = (char *)__get_free_page(GFP_KERNEL);
527 if (!buf)
528 return -ENOMEM;
529 if (copy_from_user(buf, user_buf, count)) {
530 free_page((unsigned long) buf);
531 return -EFAULT;
532 }
533 /* NULL-terminate and remove enter */
534 buf[count] = '\0';
535 strim(buf);
536
537 type = parse_cp_type(buf, count);
538 free_page((unsigned long) buf);
539 if (type == NONE)
540 return -EINVAL;
541
542 printk(KERN_INFO "lkdtm: Performing direct entry %s\n",
543 cp_type_to_str(type));
544 lkdtm_do_action(type);
545 *off += count;
546
547 return count;
548}
549
550struct crash_entry {
551 const char *name;
552 const struct file_operations fops;
553};
554
555static const struct crash_entry crash_entries[] = {
556 {"DIRECT", {.read = lkdtm_debugfs_read,
557 .open = lkdtm_debugfs_open,
558 .write = direct_entry} },
559 {"INT_HARDWARE_ENTRY", {.read = lkdtm_debugfs_read,
560 .open = lkdtm_debugfs_open,
561 .write = int_hardware_entry} },
562 {"INT_HW_IRQ_EN", {.read = lkdtm_debugfs_read,
563 .open = lkdtm_debugfs_open,
564 .write = int_hw_irq_en} },
565 {"INT_TASKLET_ENTRY", {.read = lkdtm_debugfs_read,
566 .open = lkdtm_debugfs_open,
567 .write = int_tasklet_entry} },
568 {"FS_DEVRW", {.read = lkdtm_debugfs_read,
569 .open = lkdtm_debugfs_open,
570 .write = fs_devrw_entry} },
571 {"MEM_SWAPOUT", {.read = lkdtm_debugfs_read,
572 .open = lkdtm_debugfs_open,
573 .write = mem_swapout_entry} },
574 {"TIMERADD", {.read = lkdtm_debugfs_read,
575 .open = lkdtm_debugfs_open,
576 .write = timeradd_entry} },
577 {"SCSI_DISPATCH_CMD", {.read = lkdtm_debugfs_read,
578 .open = lkdtm_debugfs_open,
579 .write = scsi_dispatch_cmd_entry} },
580 {"IDE_CORE_CP", {.read = lkdtm_debugfs_read,
581 .open = lkdtm_debugfs_open,
582 .write = ide_core_cp_entry} },
583};
584
585static struct dentry *lkdtm_debugfs_root;
586
587static int __init lkdtm_module_init(void)
588{
589 int ret = -EINVAL;
590 int n_debugfs_entries = 1; /* Assume only the direct entry */
591 int i;
592
593 /* Register debugfs interface */
594 lkdtm_debugfs_root = debugfs_create_dir("provoke-crash", NULL);
595 if (!lkdtm_debugfs_root) {
596 printk(KERN_ERR "lkdtm: creating root dir failed\n");
597 return -ENODEV;
598 }
599
600#ifdef CONFIG_KPROBES
601 n_debugfs_entries = ARRAY_SIZE(crash_entries);
602#endif
603
604 for (i = 0; i < n_debugfs_entries; i++) {
605 const struct crash_entry *cur = &crash_entries[i];
606 struct dentry *de;
607
608 de = debugfs_create_file(cur->name, 0644, lkdtm_debugfs_root,
609 NULL, &cur->fops);
610 if (de == NULL) {
611 printk(KERN_ERR "lkdtm: could not create %s\n",
612 cur->name);
613 goto out_err;
614 }
615 }
616
617 if (lkdtm_parse_commandline() == -EINVAL) {
618 printk(KERN_INFO "lkdtm: Invalid command\n");
619 goto out_err;
620 }
621
622 if (cpoint != INVALID && cptype != NONE) {
623 ret = lkdtm_register_cpoint(cpoint);
624 if (ret < 0) {
625 printk(KERN_INFO "lkdtm: Invalid crash point %d\n",
626 cpoint);
627 goto out_err;
628 }
629 printk(KERN_INFO "lkdtm: Crash point %s of type %s registered\n",
630 cpoint_name, cpoint_type);
631 } else {
632 printk(KERN_INFO "lkdtm: No crash points registered, enable through debugfs\n");
329 } 633 }
330 634
331 printk(KERN_INFO "lkdtm : Crash point %s of type %s registered\n",
332 cpoint_name, cpoint_type);
333 return 0; 635 return 0;
636
637out_err:
638 debugfs_remove_recursive(lkdtm_debugfs_root);
639 return ret;
334} 640}
335 641
336static void __exit lkdtm_module_exit(void) 642static void __exit lkdtm_module_exit(void)
337{ 643{
338 unregister_jprobe(&lkdtm); 644 debugfs_remove_recursive(lkdtm_debugfs_root);
339 printk(KERN_INFO "lkdtm : Crash point unregistered\n"); 645
646 unregister_jprobe(&lkdtm);
647 printk(KERN_INFO "lkdtm: Crash point unregistered\n");
340} 648}
341 649
342module_init(lkdtm_module_init); 650module_init(lkdtm_module_init);
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 5e3407d997b2..b520ec1f33c5 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -864,8 +864,7 @@ config DEBUG_FORCE_WEAK_PER_CPU
864 864
865config LKDTM 865config LKDTM
866 tristate "Linux Kernel Dump Test Tool Module" 866 tristate "Linux Kernel Dump Test Tool Module"
867 depends on DEBUG_KERNEL 867 depends on DEBUG_FS
868 depends on KPROBES
869 depends on BLOCK 868 depends on BLOCK
870 default n 869 default n
871 help 870 help
@@ -876,7 +875,7 @@ config LKDTM
876 called lkdtm. 875 called lkdtm.
877 876
878 Documentation on how to use the module can be found in 877 Documentation on how to use the module can be found in
879 drivers/misc/lkdtm.c 878 Documentation/fault-injection/provoke-crashes.txt
880 879
881config FAULT_INJECTION 880config FAULT_INJECTION
882 bool "Fault-injection framework" 881 bool "Fault-injection framework"