aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorLen Brown <len.brown@intel.com>2006-12-20 02:53:13 -0500
committerLen Brown <len.brown@intel.com>2006-12-20 02:53:13 -0500
commit9774f3384125912eb491ca77f77907324db3ed05 (patch)
tree0bdc7486e911dd9e955b41283ee19ac74521f7bd /kernel
parent3be11c8f4f2fa194834c2e83540f34da442b8977 (diff)
parentf238085415c56618e042252894f2fcc971add645 (diff)
merge linus into test branch
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Makefile1
-rw-r--r--kernel/unwind.c1305
-rw-r--r--kernel/workqueue.c16
3 files changed, 8 insertions, 1314 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index 5e3f3b75563a..14f4d45e0ae9 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -31,7 +31,6 @@ obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
31obj-$(CONFIG_UID16) += uid16.o 31obj-$(CONFIG_UID16) += uid16.o
32obj-$(CONFIG_MODULES) += module.o 32obj-$(CONFIG_MODULES) += module.o
33obj-$(CONFIG_KALLSYMS) += kallsyms.o 33obj-$(CONFIG_KALLSYMS) += kallsyms.o
34obj-$(CONFIG_STACK_UNWIND) += unwind.o
35obj-$(CONFIG_PM) += power/ 34obj-$(CONFIG_PM) += power/
36obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o 35obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o
37obj-$(CONFIG_KEXEC) += kexec.o 36obj-$(CONFIG_KEXEC) += kexec.o
diff --git a/kernel/unwind.c b/kernel/unwind.c
deleted file mode 100644
index 09c261329249..000000000000
--- a/kernel/unwind.c
+++ /dev/null
@@ -1,1305 +0,0 @@
1/*
2 * Copyright (C) 2002-2006 Novell, Inc.
3 * Jan Beulich <jbeulich@novell.com>
4 * This code is released under version 2 of the GNU GPL.
5 *
6 * A simple API for unwinding kernel stacks. This is used for
7 * debugging and error reporting purposes. The kernel doesn't need
8 * full-blown stack unwinding with all the bells and whistles, so there
9 * is not much point in implementing the full Dwarf2 unwind API.
10 */
11
12#include <linux/unwind.h>
13#include <linux/module.h>
14#include <linux/bootmem.h>
15#include <linux/sort.h>
16#include <linux/stop_machine.h>
17#include <linux/uaccess.h>
18#include <asm/sections.h>
19#include <asm/uaccess.h>
20#include <asm/unaligned.h>
21
22extern const char __start_unwind[], __end_unwind[];
23extern const u8 __start_unwind_hdr[], __end_unwind_hdr[];
24
25#define MAX_STACK_DEPTH 8
26
27#define EXTRA_INFO(f) { \
28 BUILD_BUG_ON_ZERO(offsetof(struct unwind_frame_info, f) \
29 % FIELD_SIZEOF(struct unwind_frame_info, f)) \
30 + offsetof(struct unwind_frame_info, f) \
31 / FIELD_SIZEOF(struct unwind_frame_info, f), \
32 FIELD_SIZEOF(struct unwind_frame_info, f) \
33 }
34#define PTREGS_INFO(f) EXTRA_INFO(regs.f)
35
36static const struct {
37 unsigned offs:BITS_PER_LONG / 2;
38 unsigned width:BITS_PER_LONG / 2;
39} reg_info[] = {
40 UNW_REGISTER_INFO
41};
42
43#undef PTREGS_INFO
44#undef EXTRA_INFO
45
46#ifndef REG_INVALID
47#define REG_INVALID(r) (reg_info[r].width == 0)
48#endif
49
50#define DW_CFA_nop 0x00
51#define DW_CFA_set_loc 0x01
52#define DW_CFA_advance_loc1 0x02
53#define DW_CFA_advance_loc2 0x03
54#define DW_CFA_advance_loc4 0x04
55#define DW_CFA_offset_extended 0x05
56#define DW_CFA_restore_extended 0x06
57#define DW_CFA_undefined 0x07
58#define DW_CFA_same_value 0x08
59#define DW_CFA_register 0x09
60#define DW_CFA_remember_state 0x0a
61#define DW_CFA_restore_state 0x0b
62#define DW_CFA_def_cfa 0x0c
63#define DW_CFA_def_cfa_register 0x0d
64#define DW_CFA_def_cfa_offset 0x0e
65#define DW_CFA_def_cfa_expression 0x0f
66#define DW_CFA_expression 0x10
67#define DW_CFA_offset_extended_sf 0x11
68#define DW_CFA_def_cfa_sf 0x12
69#define DW_CFA_def_cfa_offset_sf 0x13
70#define DW_CFA_val_offset 0x14
71#define DW_CFA_val_offset_sf 0x15
72#define DW_CFA_val_expression 0x16
73#define DW_CFA_lo_user 0x1c
74#define DW_CFA_GNU_window_save 0x2d
75#define DW_CFA_GNU_args_size 0x2e
76#define DW_CFA_GNU_negative_offset_extended 0x2f
77#define DW_CFA_hi_user 0x3f
78
79#define DW_EH_PE_FORM 0x07
80#define DW_EH_PE_native 0x00
81#define DW_EH_PE_leb128 0x01
82#define DW_EH_PE_data2 0x02
83#define DW_EH_PE_data4 0x03
84#define DW_EH_PE_data8 0x04
85#define DW_EH_PE_signed 0x08
86#define DW_EH_PE_ADJUST 0x70
87#define DW_EH_PE_abs 0x00
88#define DW_EH_PE_pcrel 0x10
89#define DW_EH_PE_textrel 0x20
90#define DW_EH_PE_datarel 0x30
91#define DW_EH_PE_funcrel 0x40
92#define DW_EH_PE_aligned 0x50
93#define DW_EH_PE_indirect 0x80
94#define DW_EH_PE_omit 0xff
95
96typedef unsigned long uleb128_t;
97typedef signed long sleb128_t;
98#define sleb128abs __builtin_labs
99
100static struct unwind_table {
101 struct {
102 unsigned long pc;
103 unsigned long range;
104 } core, init;
105 const void *address;
106 unsigned long size;
107 const unsigned char *header;
108 unsigned long hdrsz;
109 struct unwind_table *link;
110 const char *name;
111} root_table;
112
113struct unwind_item {
114 enum item_location {
115 Nowhere,
116 Memory,
117 Register,
118 Value
119 } where;
120 uleb128_t value;
121};
122
123struct unwind_state {
124 uleb128_t loc, org;
125 const u8 *cieStart, *cieEnd;
126 uleb128_t codeAlign;
127 sleb128_t dataAlign;
128 struct cfa {
129 uleb128_t reg, offs;
130 } cfa;
131 struct unwind_item regs[ARRAY_SIZE(reg_info)];
132 unsigned stackDepth:8;
133 unsigned version:8;
134 const u8 *label;
135 const u8 *stack[MAX_STACK_DEPTH];
136};
137
138static const struct cfa badCFA = { ARRAY_SIZE(reg_info), 1 };
139
140static unsigned unwind_debug;
141static int __init unwind_debug_setup(char *s)
142{
143 unwind_debug = simple_strtoul(s, NULL, 0);
144 return 1;
145}
146__setup("unwind_debug=", unwind_debug_setup);
147#define dprintk(lvl, fmt, args...) \
148 ((void)(lvl > unwind_debug \
149 || printk(KERN_DEBUG "unwind: " fmt "\n", ##args)))
150
151static struct unwind_table *find_table(unsigned long pc)
152{
153 struct unwind_table *table;
154
155 for (table = &root_table; table; table = table->link)
156 if ((pc >= table->core.pc
157 && pc < table->core.pc + table->core.range)
158 || (pc >= table->init.pc
159 && pc < table->init.pc + table->init.range))
160 break;
161
162 return table;
163}
164
165static unsigned long read_pointer(const u8 **pLoc,
166 const void *end,
167 signed ptrType,
168 unsigned long text_base,
169 unsigned long data_base);
170
171static void init_unwind_table(struct unwind_table *table,
172 const char *name,
173 const void *core_start,
174 unsigned long core_size,
175 const void *init_start,
176 unsigned long init_size,
177 const void *table_start,
178 unsigned long table_size,
179 const u8 *header_start,
180 unsigned long header_size)
181{
182 const u8 *ptr = header_start + 4;
183 const u8 *end = header_start + header_size;
184
185 table->core.pc = (unsigned long)core_start;
186 table->core.range = core_size;
187 table->init.pc = (unsigned long)init_start;
188 table->init.range = init_size;
189 table->address = table_start;
190 table->size = table_size;
191 /* See if the linker provided table looks valid. */
192 if (header_size <= 4
193 || header_start[0] != 1
194 || (void *)read_pointer(&ptr, end, header_start[1], 0, 0)
195 != table_start
196 || !read_pointer(&ptr, end, header_start[2], 0, 0)
197 || !read_pointer(&ptr, end, header_start[3], 0,
198 (unsigned long)header_start)
199 || !read_pointer(&ptr, end, header_start[3], 0,
200 (unsigned long)header_start))
201 header_start = NULL;
202 table->hdrsz = header_size;
203 smp_wmb();
204 table->header = header_start;
205 table->link = NULL;
206 table->name = name;
207}
208
209void __init unwind_init(void)
210{
211 init_unwind_table(&root_table, "kernel",
212 _text, _end - _text,
213 NULL, 0,
214 __start_unwind, __end_unwind - __start_unwind,
215 __start_unwind_hdr, __end_unwind_hdr - __start_unwind_hdr);
216}
217
218static const u32 bad_cie, not_fde;
219static const u32 *cie_for_fde(const u32 *fde, const struct unwind_table *);
220static signed fde_pointer_type(const u32 *cie);
221
222struct eh_frame_hdr_table_entry {
223 unsigned long start, fde;
224};
225
226static int cmp_eh_frame_hdr_table_entries(const void *p1, const void *p2)
227{
228 const struct eh_frame_hdr_table_entry *e1 = p1;
229 const struct eh_frame_hdr_table_entry *e2 = p2;
230
231 return (e1->start > e2->start) - (e1->start < e2->start);
232}
233
234static void swap_eh_frame_hdr_table_entries(void *p1, void *p2, int size)
235{
236 struct eh_frame_hdr_table_entry *e1 = p1;
237 struct eh_frame_hdr_table_entry *e2 = p2;
238 unsigned long v;
239
240 v = e1->start;
241 e1->start = e2->start;
242 e2->start = v;
243 v = e1->fde;
244 e1->fde = e2->fde;
245 e2->fde = v;
246}
247
248static void __init setup_unwind_table(struct unwind_table *table,
249 void *(*alloc)(unsigned long))
250{
251 const u8 *ptr;
252 unsigned long tableSize = table->size, hdrSize;
253 unsigned n;
254 const u32 *fde;
255 struct {
256 u8 version;
257 u8 eh_frame_ptr_enc;
258 u8 fde_count_enc;
259 u8 table_enc;
260 unsigned long eh_frame_ptr;
261 unsigned int fde_count;
262 struct eh_frame_hdr_table_entry table[];
263 } __attribute__((__packed__)) *header;
264
265 if (table->header)
266 return;
267
268 if (table->hdrsz)
269 printk(KERN_WARNING ".eh_frame_hdr for '%s' present but unusable\n",
270 table->name);
271
272 if (tableSize & (sizeof(*fde) - 1))
273 return;
274
275 for (fde = table->address, n = 0;
276 tableSize > sizeof(*fde) && tableSize - sizeof(*fde) >= *fde;
277 tableSize -= sizeof(*fde) + *fde, fde += 1 + *fde / sizeof(*fde)) {
278 const u32 *cie = cie_for_fde(fde, table);
279 signed ptrType;
280
281 if (cie == &not_fde)
282 continue;
283 if (cie == NULL
284 || cie == &bad_cie
285 || (ptrType = fde_pointer_type(cie)) < 0)
286 return;
287 ptr = (const u8 *)(fde + 2);
288 if (!read_pointer(&ptr,
289 (const u8 *)(fde + 1) + *fde,
290 ptrType, 0, 0))
291 return;
292 ++n;
293 }
294
295 if (tableSize || !n)
296 return;
297
298 hdrSize = 4 + sizeof(unsigned long) + sizeof(unsigned int)
299 + 2 * n * sizeof(unsigned long);
300 dprintk(2, "Binary lookup table size for %s: %lu bytes", table->name, hdrSize);
301 header = alloc(hdrSize);
302 if (!header)
303 return;
304 header->version = 1;
305 header->eh_frame_ptr_enc = DW_EH_PE_abs|DW_EH_PE_native;
306 header->fde_count_enc = DW_EH_PE_abs|DW_EH_PE_data4;
307 header->table_enc = DW_EH_PE_abs|DW_EH_PE_native;
308 put_unaligned((unsigned long)table->address, &header->eh_frame_ptr);
309 BUILD_BUG_ON(offsetof(typeof(*header), fde_count)
310 % __alignof(typeof(header->fde_count)));
311 header->fde_count = n;
312
313 BUILD_BUG_ON(offsetof(typeof(*header), table)
314 % __alignof(typeof(*header->table)));
315 for (fde = table->address, tableSize = table->size, n = 0;
316 tableSize;
317 tableSize -= sizeof(*fde) + *fde, fde += 1 + *fde / sizeof(*fde)) {
318 const u32 *cie = fde + 1 - fde[1] / sizeof(*fde);
319
320 if (!fde[1])
321 continue; /* this is a CIE */
322 ptr = (const u8 *)(fde + 2);
323 header->table[n].start = read_pointer(&ptr,
324 (const u8 *)(fde + 1) + *fde,
325 fde_pointer_type(cie), 0, 0);
326 header->table[n].fde = (unsigned long)fde;
327 ++n;
328 }
329 WARN_ON(n != header->fde_count);
330
331 sort(header->table,
332 n,
333 sizeof(*header->table),
334 cmp_eh_frame_hdr_table_entries,
335 swap_eh_frame_hdr_table_entries);
336
337 table->hdrsz = hdrSize;
338 smp_wmb();
339 table->header = (const void *)header;
340}
341
342static void *__init balloc(unsigned long sz)
343{
344 return __alloc_bootmem_nopanic(sz,
345 sizeof(unsigned int),
346 __pa(MAX_DMA_ADDRESS));
347}
348
349void __init unwind_setup(void)
350{
351 setup_unwind_table(&root_table, balloc);
352}
353
354#ifdef CONFIG_MODULES
355
356static struct unwind_table *last_table;
357
358/* Must be called with module_mutex held. */
359void *unwind_add_table(struct module *module,
360 const void *table_start,
361 unsigned long table_size)
362{
363 struct unwind_table *table;
364
365 if (table_size <= 0)
366 return NULL;
367
368 table = kmalloc(sizeof(*table), GFP_KERNEL);
369 if (!table)
370 return NULL;
371
372 init_unwind_table(table, module->name,
373 module->module_core, module->core_size,
374 module->module_init, module->init_size,
375 table_start, table_size,
376 NULL, 0);
377
378 if (last_table)
379 last_table->link = table;
380 else
381 root_table.link = table;
382 last_table = table;
383
384 return table;
385}
386
387struct unlink_table_info
388{
389 struct unwind_table *table;
390 int init_only;
391};
392
393static int unlink_table(void *arg)
394{
395 struct unlink_table_info *info = arg;
396 struct unwind_table *table = info->table, *prev;
397
398 for (prev = &root_table; prev->link && prev->link != table; prev = prev->link)
399 ;
400
401 if (prev->link) {
402 if (info->init_only) {
403 table->init.pc = 0;
404 table->init.range = 0;
405 info->table = NULL;
406 } else {
407 prev->link = table->link;
408 if (!prev->link)
409 last_table = prev;
410 }
411 } else
412 info->table = NULL;
413
414 return 0;
415}
416
417/* Must be called with module_mutex held. */
418void unwind_remove_table(void *handle, int init_only)
419{
420 struct unwind_table *table = handle;
421 struct unlink_table_info info;
422
423 if (!table || table == &root_table)
424 return;
425
426 if (init_only && table == last_table) {
427 table->init.pc = 0;
428 table->init.range = 0;
429 return;
430 }
431
432 info.table = table;
433 info.init_only = init_only;
434 stop_machine_run(unlink_table, &info, NR_CPUS);
435
436 if (info.table)
437 kfree(table);
438}
439
440#endif /* CONFIG_MODULES */
441
442static uleb128_t get_uleb128(const u8 **pcur, const u8 *end)
443{
444 const u8 *cur = *pcur;
445 uleb128_t value;
446 unsigned shift;
447
448 for (shift = 0, value = 0; cur < end; shift += 7) {
449 if (shift + 7 > 8 * sizeof(value)
450 && (*cur & 0x7fU) >= (1U << (8 * sizeof(value) - shift))) {
451 cur = end + 1;
452 break;
453 }
454 value |= (uleb128_t)(*cur & 0x7f) << shift;
455 if (!(*cur++ & 0x80))
456 break;
457 }
458 *pcur = cur;
459
460 return value;
461}
462
463static sleb128_t get_sleb128(const u8 **pcur, const u8 *end)
464{
465 const u8 *cur = *pcur;
466 sleb128_t value;
467 unsigned shift;
468
469 for (shift = 0, value = 0; cur < end; shift += 7) {
470 if (shift + 7 > 8 * sizeof(value)
471 && (*cur & 0x7fU) >= (1U << (8 * sizeof(value) - shift))) {
472 cur = end + 1;
473 break;
474 }
475 value |= (sleb128_t)(*cur & 0x7f) << shift;
476 if (!(*cur & 0x80)) {
477 value |= -(*cur++ & 0x40) << shift;
478 break;
479 }
480 }
481 *pcur = cur;
482
483 return value;
484}
485
486static const u32 *cie_for_fde(const u32 *fde, const struct unwind_table *table)
487{
488 const u32 *cie;
489
490 if (!*fde || (*fde & (sizeof(*fde) - 1)))
491 return &bad_cie;
492 if (!fde[1])
493 return &not_fde; /* this is a CIE */
494 if ((fde[1] & (sizeof(*fde) - 1))
495 || fde[1] > (unsigned long)(fde + 1) - (unsigned long)table->address)
496 return NULL; /* this is not a valid FDE */
497 cie = fde + 1 - fde[1] / sizeof(*fde);
498 if (*cie <= sizeof(*cie) + 4
499 || *cie >= fde[1] - sizeof(*fde)
500 || (*cie & (sizeof(*cie) - 1))
501 || cie[1])
502 return NULL; /* this is not a (valid) CIE */
503 return cie;
504}
505
506static unsigned long read_pointer(const u8 **pLoc,
507 const void *end,
508 signed ptrType,
509 unsigned long text_base,
510 unsigned long data_base)
511{
512 unsigned long value = 0;
513 union {
514 const u8 *p8;
515 const u16 *p16u;
516 const s16 *p16s;
517 const u32 *p32u;
518 const s32 *p32s;
519 const unsigned long *pul;
520 } ptr;
521
522 if (ptrType < 0 || ptrType == DW_EH_PE_omit) {
523 dprintk(1, "Invalid pointer encoding %02X (%p,%p).", ptrType, *pLoc, end);
524 return 0;
525 }
526 ptr.p8 = *pLoc;
527 switch(ptrType & DW_EH_PE_FORM) {
528 case DW_EH_PE_data2:
529 if (end < (const void *)(ptr.p16u + 1)) {
530 dprintk(1, "Data16 overrun (%p,%p).", ptr.p8, end);
531 return 0;
532 }
533 if(ptrType & DW_EH_PE_signed)
534 value = get_unaligned(ptr.p16s++);
535 else
536 value = get_unaligned(ptr.p16u++);
537 break;
538 case DW_EH_PE_data4:
539#ifdef CONFIG_64BIT
540 if (end < (const void *)(ptr.p32u + 1)) {
541 dprintk(1, "Data32 overrun (%p,%p).", ptr.p8, end);
542 return 0;
543 }
544 if(ptrType & DW_EH_PE_signed)
545 value = get_unaligned(ptr.p32s++);
546 else
547 value = get_unaligned(ptr.p32u++);
548 break;
549 case DW_EH_PE_data8:
550 BUILD_BUG_ON(sizeof(u64) != sizeof(value));
551#else
552 BUILD_BUG_ON(sizeof(u32) != sizeof(value));
553#endif
554 case DW_EH_PE_native:
555 if (end < (const void *)(ptr.pul + 1)) {
556 dprintk(1, "DataUL overrun (%p,%p).", ptr.p8, end);
557 return 0;
558 }
559 value = get_unaligned(ptr.pul++);
560 break;
561 case DW_EH_PE_leb128:
562 BUILD_BUG_ON(sizeof(uleb128_t) > sizeof(value));
563 value = ptrType & DW_EH_PE_signed
564 ? get_sleb128(&ptr.p8, end)
565 : get_uleb128(&ptr.p8, end);
566 if ((const void *)ptr.p8 > end) {
567 dprintk(1, "DataLEB overrun (%p,%p).", ptr.p8, end);
568 return 0;
569 }
570 break;
571 default:
572 dprintk(2, "Cannot decode pointer type %02X (%p,%p).",
573 ptrType, ptr.p8, end);
574 return 0;
575 }
576 switch(ptrType & DW_EH_PE_ADJUST) {
577 case DW_EH_PE_abs:
578 break;
579 case DW_EH_PE_pcrel:
580 value += (unsigned long)*pLoc;
581 break;
582 case DW_EH_PE_textrel:
583 if (likely(text_base)) {
584 value += text_base;
585 break;
586 }
587 dprintk(2, "Text-relative encoding %02X (%p,%p), but zero text base.",
588 ptrType, *pLoc, end);
589 return 0;
590 case DW_EH_PE_datarel:
591 if (likely(data_base)) {
592 value += data_base;
593 break;
594 }
595 dprintk(2, "Data-relative encoding %02X (%p,%p), but zero data base.",
596 ptrType, *pLoc, end);
597 return 0;
598 default:
599 dprintk(2, "Cannot adjust pointer type %02X (%p,%p).",
600 ptrType, *pLoc, end);
601 return 0;
602 }
603 if ((ptrType & DW_EH_PE_indirect)
604 && probe_kernel_address((unsigned long *)value, value)) {
605 dprintk(1, "Cannot read indirect value %lx (%p,%p).",
606 value, *pLoc, end);
607 return 0;
608 }
609 *pLoc = ptr.p8;
610
611 return value;
612}
613
614static signed fde_pointer_type(const u32 *cie)
615{
616 const u8 *ptr = (const u8 *)(cie + 2);
617 unsigned version = *ptr;
618
619 if (version != 1)
620 return -1; /* unsupported */
621 if (*++ptr) {
622 const char *aug;
623 const u8 *end = (const u8 *)(cie + 1) + *cie;
624 uleb128_t len;
625
626 /* check if augmentation size is first (and thus present) */
627 if (*ptr != 'z')
628 return -1;
629 /* check if augmentation string is nul-terminated */
630 if ((ptr = memchr(aug = (const void *)ptr, 0, end - ptr)) == NULL)
631 return -1;
632 ++ptr; /* skip terminator */
633 get_uleb128(&ptr, end); /* skip code alignment */
634 get_sleb128(&ptr, end); /* skip data alignment */
635 /* skip return address column */
636 version <= 1 ? (void)++ptr : (void)get_uleb128(&ptr, end);
637 len = get_uleb128(&ptr, end); /* augmentation length */
638 if (ptr + len < ptr || ptr + len > end)
639 return -1;
640 end = ptr + len;
641 while (*++aug) {
642 if (ptr >= end)
643 return -1;
644 switch(*aug) {
645 case 'L':
646 ++ptr;
647 break;
648 case 'P': {
649 signed ptrType = *ptr++;
650
651 if (!read_pointer(&ptr, end, ptrType, 0, 0)
652 || ptr > end)
653 return -1;
654 }
655 break;
656 case 'R':
657 return *ptr;
658 default:
659 return -1;
660 }
661 }
662 }
663 return DW_EH_PE_native|DW_EH_PE_abs;
664}
665
666static int advance_loc(unsigned long delta, struct unwind_state *state)
667{
668 state->loc += delta * state->codeAlign;
669
670 return delta > 0;
671}
672
673static void set_rule(uleb128_t reg,
674 enum item_location where,
675 uleb128_t value,
676 struct unwind_state *state)
677{
678 if (reg < ARRAY_SIZE(state->regs)) {
679 state->regs[reg].where = where;
680 state->regs[reg].value = value;
681 }
682}
683
684static int processCFI(const u8 *start,
685 const u8 *end,
686 unsigned long targetLoc,
687 signed ptrType,
688 struct unwind_state *state)
689{
690 union {
691 const u8 *p8;
692 const u16 *p16;
693 const u32 *p32;
694 } ptr;
695 int result = 1;
696
697 if (start != state->cieStart) {
698 state->loc = state->org;
699 result = processCFI(state->cieStart, state->cieEnd, 0, ptrType, state);
700 if (targetLoc == 0 && state->label == NULL)
701 return result;
702 }
703 for (ptr.p8 = start; result && ptr.p8 < end; ) {
704 switch(*ptr.p8 >> 6) {
705 uleb128_t value;
706
707 case 0:
708 switch(*ptr.p8++) {
709 case DW_CFA_nop:
710 break;
711 case DW_CFA_set_loc:
712 state->loc = read_pointer(&ptr.p8, end, ptrType, 0, 0);
713 if (state->loc == 0)
714 result = 0;
715 break;
716 case DW_CFA_advance_loc1:
717 result = ptr.p8 < end && advance_loc(*ptr.p8++, state);
718 break;
719 case DW_CFA_advance_loc2:
720 result = ptr.p8 <= end + 2
721 && advance_loc(*ptr.p16++, state);
722 break;
723 case DW_CFA_advance_loc4:
724 result = ptr.p8 <= end + 4
725 && advance_loc(*ptr.p32++, state);
726 break;
727 case DW_CFA_offset_extended:
728 value = get_uleb128(&ptr.p8, end);
729 set_rule(value, Memory, get_uleb128(&ptr.p8, end), state);
730 break;
731 case DW_CFA_val_offset:
732 value = get_uleb128(&ptr.p8, end);
733 set_rule(value, Value, get_uleb128(&ptr.p8, end), state);
734 break;
735 case DW_CFA_offset_extended_sf:
736 value = get_uleb128(&ptr.p8, end);
737 set_rule(value, Memory, get_sleb128(&ptr.p8, end), state);
738 break;
739 case DW_CFA_val_offset_sf:
740 value = get_uleb128(&ptr.p8, end);
741 set_rule(value, Value, get_sleb128(&ptr.p8, end), state);
742 break;
743 case DW_CFA_restore_extended:
744 case DW_CFA_undefined:
745 case DW_CFA_same_value:
746 set_rule(get_uleb128(&ptr.p8, end), Nowhere, 0, state);
747 break;
748 case DW_CFA_register:
749 value = get_uleb128(&ptr.p8, end);
750 set_rule(value,
751 Register,
752 get_uleb128(&ptr.p8, end), state);
753 break;
754 case DW_CFA_remember_state:
755 if (ptr.p8 == state->label) {
756 state->label = NULL;
757 return 1;
758 }
759 if (state->stackDepth >= MAX_STACK_DEPTH) {
760 dprintk(1, "State stack overflow (%p,%p).", ptr.p8, end);
761 return 0;
762 }
763 state->stack[state->stackDepth++] = ptr.p8;
764 break;
765 case DW_CFA_restore_state:
766 if (state->stackDepth) {
767 const uleb128_t loc = state->loc;
768 const u8 *label = state->label;
769
770 state->label = state->stack[state->stackDepth - 1];
771 memcpy(&state->cfa, &badCFA, sizeof(state->cfa));
772 memset(state->regs, 0, sizeof(state->regs));
773 state->stackDepth = 0;
774 result = processCFI(start, end, 0, ptrType, state);
775 state->loc = loc;
776 state->label = label;
777 } else {
778 dprintk(1, "State stack underflow (%p,%p).", ptr.p8, end);
779 return 0;
780 }
781 break;
782 case DW_CFA_def_cfa:
783 state->cfa.reg = get_uleb128(&ptr.p8, end);
784 /*nobreak*/
785 case DW_CFA_def_cfa_offset:
786 state->cfa.offs = get_uleb128(&ptr.p8, end);
787 break;
788 case DW_CFA_def_cfa_sf:
789 state->cfa.reg = get_uleb128(&ptr.p8, end);
790 /*nobreak*/
791 case DW_CFA_def_cfa_offset_sf:
792 state->cfa.offs = get_sleb128(&ptr.p8, end)
793 * state->dataAlign;
794 break;
795 case DW_CFA_def_cfa_register:
796 state->cfa.reg = get_uleb128(&ptr.p8, end);
797 break;
798 /*todo case DW_CFA_def_cfa_expression: */
799 /*todo case DW_CFA_expression: */
800 /*todo case DW_CFA_val_expression: */
801 case DW_CFA_GNU_args_size:
802 get_uleb128(&ptr.p8, end);
803 break;
804 case DW_CFA_GNU_negative_offset_extended:
805 value = get_uleb128(&ptr.p8, end);
806 set_rule(value,
807 Memory,
808 (uleb128_t)0 - get_uleb128(&ptr.p8, end), state);
809 break;
810 case DW_CFA_GNU_window_save:
811 default:
812 dprintk(1, "Unrecognized CFI op %02X (%p,%p).", ptr.p8[-1], ptr.p8 - 1, end);
813 result = 0;
814 break;
815 }
816 break;
817 case 1:
818 result = advance_loc(*ptr.p8++ & 0x3f, state);
819 break;
820 case 2:
821 value = *ptr.p8++ & 0x3f;
822 set_rule(value, Memory, get_uleb128(&ptr.p8, end), state);
823 break;
824 case 3:
825 set_rule(*ptr.p8++ & 0x3f, Nowhere, 0, state);
826 break;
827 }
828 if (ptr.p8 > end) {
829 dprintk(1, "Data overrun (%p,%p).", ptr.p8, end);
830 result = 0;
831 }
832 if (result && targetLoc != 0 && targetLoc < state->loc)
833 return 1;
834 }
835
836 if (result && ptr.p8 < end)
837 dprintk(1, "Data underrun (%p,%p).", ptr.p8, end);
838
839 return result
840 && ptr.p8 == end
841 && (targetLoc == 0
842 || (/*todo While in theory this should apply, gcc in practice omits
843 everything past the function prolog, and hence the location
844 never reaches the end of the function.
845 targetLoc < state->loc &&*/ state->label == NULL));
846}
847
848/* Unwind to previous to frame. Returns 0 if successful, negative
849 * number in case of an error. */
850int unwind(struct unwind_frame_info *frame)
851{
852#define FRAME_REG(r, t) (((t *)frame)[reg_info[r].offs])
853 const u32 *fde = NULL, *cie = NULL;
854 const u8 *ptr = NULL, *end = NULL;
855 unsigned long pc = UNW_PC(frame) - frame->call_frame, sp;
856 unsigned long startLoc = 0, endLoc = 0, cfa;
857 unsigned i;
858 signed ptrType = -1;
859 uleb128_t retAddrReg = 0;
860 const struct unwind_table *table;
861 struct unwind_state state;
862
863 if (UNW_PC(frame) == 0)
864 return -EINVAL;
865 if ((table = find_table(pc)) != NULL
866 && !(table->size & (sizeof(*fde) - 1))) {
867 const u8 *hdr = table->header;
868 unsigned long tableSize;
869
870 smp_rmb();
871 if (hdr && hdr[0] == 1) {
872 switch(hdr[3] & DW_EH_PE_FORM) {
873 case DW_EH_PE_native: tableSize = sizeof(unsigned long); break;
874 case DW_EH_PE_data2: tableSize = 2; break;
875 case DW_EH_PE_data4: tableSize = 4; break;
876 case DW_EH_PE_data8: tableSize = 8; break;
877 default: tableSize = 0; break;
878 }
879 ptr = hdr + 4;
880 end = hdr + table->hdrsz;
881 if (tableSize
882 && read_pointer(&ptr, end, hdr[1], 0, 0)
883 == (unsigned long)table->address
884 && (i = read_pointer(&ptr, end, hdr[2], 0, 0)) > 0
885 && i == (end - ptr) / (2 * tableSize)
886 && !((end - ptr) % (2 * tableSize))) {
887 do {
888 const u8 *cur = ptr + (i / 2) * (2 * tableSize);
889
890 startLoc = read_pointer(&cur,
891 cur + tableSize,
892 hdr[3], 0,
893 (unsigned long)hdr);
894 if (pc < startLoc)
895 i /= 2;
896 else {
897 ptr = cur - tableSize;
898 i = (i + 1) / 2;
899 }
900 } while (startLoc && i > 1);
901 if (i == 1
902 && (startLoc = read_pointer(&ptr,
903 ptr + tableSize,
904 hdr[3], 0,
905 (unsigned long)hdr)) != 0
906 && pc >= startLoc)
907 fde = (void *)read_pointer(&ptr,
908 ptr + tableSize,
909 hdr[3], 0,
910 (unsigned long)hdr);
911 }
912 }
913 if(hdr && !fde)
914 dprintk(3, "Binary lookup for %lx failed.", pc);
915
916 if (fde != NULL) {
917 cie = cie_for_fde(fde, table);
918 ptr = (const u8 *)(fde + 2);
919 if(cie != NULL
920 && cie != &bad_cie
921 && cie != &not_fde
922 && (ptrType = fde_pointer_type(cie)) >= 0
923 && read_pointer(&ptr,
924 (const u8 *)(fde + 1) + *fde,
925 ptrType, 0, 0) == startLoc) {
926 if (!(ptrType & DW_EH_PE_indirect))
927 ptrType &= DW_EH_PE_FORM|DW_EH_PE_signed;
928 endLoc = startLoc
929 + read_pointer(&ptr,
930 (const u8 *)(fde + 1) + *fde,
931 ptrType, 0, 0);
932 if(pc >= endLoc)
933 fde = NULL;
934 } else
935 fde = NULL;
936 if(!fde)
937 dprintk(1, "Binary lookup result for %lx discarded.", pc);
938 }
939 if (fde == NULL) {
940 for (fde = table->address, tableSize = table->size;
941 cie = NULL, tableSize > sizeof(*fde)
942 && tableSize - sizeof(*fde) >= *fde;
943 tableSize -= sizeof(*fde) + *fde,
944 fde += 1 + *fde / sizeof(*fde)) {
945 cie = cie_for_fde(fde, table);
946 if (cie == &bad_cie) {
947 cie = NULL;
948 break;
949 }
950 if (cie == NULL
951 || cie == &not_fde
952 || (ptrType = fde_pointer_type(cie)) < 0)
953 continue;
954 ptr = (const u8 *)(fde + 2);
955 startLoc = read_pointer(&ptr,
956 (const u8 *)(fde + 1) + *fde,
957 ptrType, 0, 0);
958 if (!startLoc)
959 continue;
960 if (!(ptrType & DW_EH_PE_indirect))
961 ptrType &= DW_EH_PE_FORM|DW_EH_PE_signed;
962 endLoc = startLoc
963 + read_pointer(&ptr,
964 (const u8 *)(fde + 1) + *fde,
965 ptrType, 0, 0);
966 if (pc >= startLoc && pc < endLoc)
967 break;
968 }
969 if(!fde)
970 dprintk(3, "Linear lookup for %lx failed.", pc);
971 }
972 }
973 if (cie != NULL) {
974 memset(&state, 0, sizeof(state));
975 state.cieEnd = ptr; /* keep here temporarily */
976 ptr = (const u8 *)(cie + 2);
977 end = (const u8 *)(cie + 1) + *cie;
978 frame->call_frame = 1;
979 if ((state.version = *ptr) != 1)
980 cie = NULL; /* unsupported version */
981 else if (*++ptr) {
982 /* check if augmentation size is first (and thus present) */
983 if (*ptr == 'z') {
984 while (++ptr < end && *ptr) {
985 switch(*ptr) {
986 /* check for ignorable (or already handled)
987 * nul-terminated augmentation string */
988 case 'L':
989 case 'P':
990 case 'R':
991 continue;
992 case 'S':
993 frame->call_frame = 0;
994 continue;
995 default:
996 break;
997 }
998 break;
999 }
1000 }
1001 if (ptr >= end || *ptr)
1002 cie = NULL;
1003 }
1004 if(!cie)
1005 dprintk(1, "CIE unusable (%p,%p).", ptr, end);
1006 ++ptr;
1007 }
1008 if (cie != NULL) {
1009 /* get code aligment factor */
1010 state.codeAlign = get_uleb128(&ptr, end);
1011 /* get data aligment factor */
1012 state.dataAlign = get_sleb128(&ptr, end);
1013 if (state.codeAlign == 0 || state.dataAlign == 0 || ptr >= end)
1014 cie = NULL;
1015 else if (UNW_PC(frame) % state.codeAlign
1016 || UNW_SP(frame) % sleb128abs(state.dataAlign)) {
1017 dprintk(1, "Input pointer(s) misaligned (%lx,%lx).",
1018 UNW_PC(frame), UNW_SP(frame));
1019 return -EPERM;
1020 } else {
1021 retAddrReg = state.version <= 1 ? *ptr++ : get_uleb128(&ptr, end);
1022 /* skip augmentation */
1023 if (((const char *)(cie + 2))[1] == 'z') {
1024 uleb128_t augSize = get_uleb128(&ptr, end);
1025
1026 ptr += augSize;
1027 }
1028 if (ptr > end
1029 || retAddrReg >= ARRAY_SIZE(reg_info)
1030 || REG_INVALID(retAddrReg)
1031 || reg_info[retAddrReg].width != sizeof(unsigned long))
1032 cie = NULL;
1033 }
1034 if(!cie)
1035 dprintk(1, "CIE validation failed (%p,%p).", ptr, end);
1036 }
1037 if (cie != NULL) {
1038 state.cieStart = ptr;
1039 ptr = state.cieEnd;
1040 state.cieEnd = end;
1041 end = (const u8 *)(fde + 1) + *fde;
1042 /* skip augmentation */
1043 if (((const char *)(cie + 2))[1] == 'z') {
1044 uleb128_t augSize = get_uleb128(&ptr, end);
1045
1046 if ((ptr += augSize) > end)
1047 fde = NULL;
1048 }
1049 if(!fde)
1050 dprintk(1, "FDE validation failed (%p,%p).", ptr, end);
1051 }
1052 if (cie == NULL || fde == NULL) {
1053#ifdef CONFIG_FRAME_POINTER
1054 unsigned long top, bottom;
1055
1056 if ((UNW_SP(frame) | UNW_FP(frame)) % sizeof(unsigned long))
1057 return -EPERM;
1058 top = STACK_TOP(frame->task);
1059 bottom = STACK_BOTTOM(frame->task);
1060# if FRAME_RETADDR_OFFSET < 0
1061 if (UNW_SP(frame) < top
1062 && UNW_FP(frame) <= UNW_SP(frame)
1063 && bottom < UNW_FP(frame)
1064# else
1065 if (UNW_SP(frame) > top
1066 && UNW_FP(frame) >= UNW_SP(frame)
1067 && bottom > UNW_FP(frame)
1068# endif
1069 && !((UNW_SP(frame) | UNW_FP(frame))
1070 & (sizeof(unsigned long) - 1))) {
1071 unsigned long link;
1072
1073 if (!probe_kernel_address(
1074 (unsigned long *)(UNW_FP(frame)
1075 + FRAME_LINK_OFFSET),
1076 link)
1077# if FRAME_RETADDR_OFFSET < 0
1078 && link > bottom && link < UNW_FP(frame)
1079# else
1080 && link > UNW_FP(frame) && link < bottom
1081# endif
1082 && !(link & (sizeof(link) - 1))
1083 && !probe_kernel_address(
1084 (unsigned long *)(UNW_FP(frame)
1085 + FRAME_RETADDR_OFFSET), UNW_PC(frame))) {
1086 UNW_SP(frame) = UNW_FP(frame) + FRAME_RETADDR_OFFSET
1087# if FRAME_RETADDR_OFFSET < 0
1088 -
1089# else
1090 +
1091# endif
1092 sizeof(UNW_PC(frame));
1093 UNW_FP(frame) = link;
1094 return 0;
1095 }
1096 }
1097#endif
1098 return -ENXIO;
1099 }
1100 state.org = startLoc;
1101 memcpy(&state.cfa, &badCFA, sizeof(state.cfa));
1102 /* process instructions */
1103 if (!processCFI(ptr, end, pc, ptrType, &state)
1104 || state.loc > endLoc
1105 || state.regs[retAddrReg].where == Nowhere
1106 || state.cfa.reg >= ARRAY_SIZE(reg_info)
1107 || reg_info[state.cfa.reg].width != sizeof(unsigned long)
1108 || FRAME_REG(state.cfa.reg, unsigned long) % sizeof(unsigned long)
1109 || state.cfa.offs % sizeof(unsigned long)) {
1110 dprintk(1, "Unusable unwind info (%p,%p).", ptr, end);
1111 return -EIO;
1112 }
1113 /* update frame */
1114#ifndef CONFIG_AS_CFI_SIGNAL_FRAME
1115 if(frame->call_frame
1116 && !UNW_DEFAULT_RA(state.regs[retAddrReg], state.dataAlign))
1117 frame->call_frame = 0;
1118#endif
1119 cfa = FRAME_REG(state.cfa.reg, unsigned long) + state.cfa.offs;
1120 startLoc = min((unsigned long)UNW_SP(frame), cfa);
1121 endLoc = max((unsigned long)UNW_SP(frame), cfa);
1122 if (STACK_LIMIT(startLoc) != STACK_LIMIT(endLoc)) {
1123 startLoc = min(STACK_LIMIT(cfa), cfa);
1124 endLoc = max(STACK_LIMIT(cfa), cfa);
1125 }
1126#ifndef CONFIG_64BIT
1127# define CASES CASE(8); CASE(16); CASE(32)
1128#else
1129# define CASES CASE(8); CASE(16); CASE(32); CASE(64)
1130#endif
1131 pc = UNW_PC(frame);
1132 sp = UNW_SP(frame);
1133 for (i = 0; i < ARRAY_SIZE(state.regs); ++i) {
1134 if (REG_INVALID(i)) {
1135 if (state.regs[i].where == Nowhere)
1136 continue;
1137 dprintk(1, "Cannot restore register %u (%d).",
1138 i, state.regs[i].where);
1139 return -EIO;
1140 }
1141 switch(state.regs[i].where) {
1142 default:
1143 break;
1144 case Register:
1145 if (state.regs[i].value >= ARRAY_SIZE(reg_info)
1146 || REG_INVALID(state.regs[i].value)
1147 || reg_info[i].width > reg_info[state.regs[i].value].width) {
1148 dprintk(1, "Cannot restore register %u from register %lu.",
1149 i, state.regs[i].value);
1150 return -EIO;
1151 }
1152 switch(reg_info[state.regs[i].value].width) {
1153#define CASE(n) \
1154 case sizeof(u##n): \
1155 state.regs[i].value = FRAME_REG(state.regs[i].value, \
1156 const u##n); \
1157 break
1158 CASES;
1159#undef CASE
1160 default:
1161 dprintk(1, "Unsupported register size %u (%lu).",
1162 reg_info[state.regs[i].value].width,
1163 state.regs[i].value);
1164 return -EIO;
1165 }
1166 break;
1167 }
1168 }
1169 for (i = 0; i < ARRAY_SIZE(state.regs); ++i) {
1170 if (REG_INVALID(i))
1171 continue;
1172 switch(state.regs[i].where) {
1173 case Nowhere:
1174 if (reg_info[i].width != sizeof(UNW_SP(frame))
1175 || &FRAME_REG(i, __typeof__(UNW_SP(frame)))
1176 != &UNW_SP(frame))
1177 continue;
1178 UNW_SP(frame) = cfa;
1179 break;
1180 case Register:
1181 switch(reg_info[i].width) {
1182#define CASE(n) case sizeof(u##n): \
1183 FRAME_REG(i, u##n) = state.regs[i].value; \
1184 break
1185 CASES;
1186#undef CASE
1187 default:
1188 dprintk(1, "Unsupported register size %u (%u).",
1189 reg_info[i].width, i);
1190 return -EIO;
1191 }
1192 break;
1193 case Value:
1194 if (reg_info[i].width != sizeof(unsigned long)) {
1195 dprintk(1, "Unsupported value size %u (%u).",
1196 reg_info[i].width, i);
1197 return -EIO;
1198 }
1199 FRAME_REG(i, unsigned long) = cfa + state.regs[i].value
1200 * state.dataAlign;
1201 break;
1202 case Memory: {
1203 unsigned long addr = cfa + state.regs[i].value
1204 * state.dataAlign;
1205
1206 if ((state.regs[i].value * state.dataAlign)
1207 % sizeof(unsigned long)
1208 || addr < startLoc
1209 || addr + sizeof(unsigned long) < addr
1210 || addr + sizeof(unsigned long) > endLoc) {
1211 dprintk(1, "Bad memory location %lx (%lx).",
1212 addr, state.regs[i].value);
1213 return -EIO;
1214 }
1215 switch(reg_info[i].width) {
1216#define CASE(n) case sizeof(u##n): \
1217 probe_kernel_address((u##n *)addr, FRAME_REG(i, u##n)); \
1218 break
1219 CASES;
1220#undef CASE
1221 default:
1222 dprintk(1, "Unsupported memory size %u (%u).",
1223 reg_info[i].width, i);
1224 return -EIO;
1225 }
1226 }
1227 break;
1228 }
1229 }
1230
1231 if (UNW_PC(frame) % state.codeAlign
1232 || UNW_SP(frame) % sleb128abs(state.dataAlign)) {
1233 dprintk(1, "Output pointer(s) misaligned (%lx,%lx).",
1234 UNW_PC(frame), UNW_SP(frame));
1235 return -EIO;
1236 }
1237 if (pc == UNW_PC(frame) && sp == UNW_SP(frame)) {
1238 dprintk(1, "No progress (%lx,%lx).", pc, sp);
1239 return -EIO;
1240 }
1241
1242 return 0;
1243#undef CASES
1244#undef FRAME_REG
1245}
1246EXPORT_SYMBOL(unwind);
1247
1248int unwind_init_frame_info(struct unwind_frame_info *info,
1249 struct task_struct *tsk,
1250 /*const*/ struct pt_regs *regs)
1251{
1252 info->task = tsk;
1253 info->call_frame = 0;
1254 arch_unw_init_frame_info(info, regs);
1255
1256 return 0;
1257}
1258EXPORT_SYMBOL(unwind_init_frame_info);
1259
1260/*
1261 * Prepare to unwind a blocked task.
1262 */
1263int unwind_init_blocked(struct unwind_frame_info *info,
1264 struct task_struct *tsk)
1265{
1266 info->task = tsk;
1267 info->call_frame = 0;
1268 arch_unw_init_blocked(info);
1269
1270 return 0;
1271}
1272EXPORT_SYMBOL(unwind_init_blocked);
1273
1274/*
1275 * Prepare to unwind the currently running thread.
1276 */
1277int unwind_init_running(struct unwind_frame_info *info,
1278 asmlinkage int (*callback)(struct unwind_frame_info *,
1279 void *arg),
1280 void *arg)
1281{
1282 info->task = current;
1283 info->call_frame = 0;
1284
1285 return arch_unwind_init_running(info, callback, arg);
1286}
1287EXPORT_SYMBOL(unwind_init_running);
1288
1289/*
1290 * Unwind until the return pointer is in user-land (or until an error
1291 * occurs). Returns 0 if successful, negative number in case of
1292 * error.
1293 */
1294int unwind_to_user(struct unwind_frame_info *info)
1295{
1296 while (!arch_unw_user_mode(info)) {
1297 int err = unwind(info);
1298
1299 if (err < 0)
1300 return err;
1301 }
1302
1303 return 0;
1304}
1305EXPORT_SYMBOL(unwind_to_user);
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index db49886bfae1..742cbbe49bdc 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -96,13 +96,13 @@ static inline void set_wq_data(struct work_struct *work, void *wq)
96 BUG_ON(!work_pending(work)); 96 BUG_ON(!work_pending(work));
97 97
98 new = (unsigned long) wq | (1UL << WORK_STRUCT_PENDING); 98 new = (unsigned long) wq | (1UL << WORK_STRUCT_PENDING);
99 new |= work->management & WORK_STRUCT_FLAG_MASK; 99 new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work);
100 work->management = new; 100 atomic_long_set(&work->data, new);
101} 101}
102 102
103static inline void *get_wq_data(struct work_struct *work) 103static inline void *get_wq_data(struct work_struct *work)
104{ 104{
105 return (void *) (work->management & WORK_STRUCT_WQ_DATA_MASK); 105 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
106} 106}
107 107
108static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work) 108static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work)
@@ -133,7 +133,7 @@ static int __run_work(struct cpu_workqueue_struct *cwq, struct work_struct *work
133 list_del_init(&work->entry); 133 list_del_init(&work->entry);
134 spin_unlock_irqrestore(&cwq->lock, flags); 134 spin_unlock_irqrestore(&cwq->lock, flags);
135 135
136 if (!test_bit(WORK_STRUCT_NOAUTOREL, &work->management)) 136 if (!test_bit(WORK_STRUCT_NOAUTOREL, work_data_bits(work)))
137 work_release(work); 137 work_release(work);
138 f(work); 138 f(work);
139 139
@@ -206,7 +206,7 @@ int fastcall queue_work(struct workqueue_struct *wq, struct work_struct *work)
206{ 206{
207 int ret = 0, cpu = get_cpu(); 207 int ret = 0, cpu = get_cpu();
208 208
209 if (!test_and_set_bit(WORK_STRUCT_PENDING, &work->management)) { 209 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
210 if (unlikely(is_single_threaded(wq))) 210 if (unlikely(is_single_threaded(wq)))
211 cpu = singlethread_cpu; 211 cpu = singlethread_cpu;
212 BUG_ON(!list_empty(&work->entry)); 212 BUG_ON(!list_empty(&work->entry));
@@ -248,7 +248,7 @@ int fastcall queue_delayed_work(struct workqueue_struct *wq,
248 if (delay == 0) 248 if (delay == 0)
249 return queue_work(wq, work); 249 return queue_work(wq, work);
250 250
251 if (!test_and_set_bit(WORK_STRUCT_PENDING, &work->management)) { 251 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
252 BUG_ON(timer_pending(timer)); 252 BUG_ON(timer_pending(timer));
253 BUG_ON(!list_empty(&work->entry)); 253 BUG_ON(!list_empty(&work->entry));
254 254
@@ -280,7 +280,7 @@ int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
280 struct timer_list *timer = &dwork->timer; 280 struct timer_list *timer = &dwork->timer;
281 struct work_struct *work = &dwork->work; 281 struct work_struct *work = &dwork->work;
282 282
283 if (!test_and_set_bit(WORK_STRUCT_PENDING, &work->management)) { 283 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
284 BUG_ON(timer_pending(timer)); 284 BUG_ON(timer_pending(timer));
285 BUG_ON(!list_empty(&work->entry)); 285 BUG_ON(!list_empty(&work->entry));
286 286
@@ -321,7 +321,7 @@ static void run_workqueue(struct cpu_workqueue_struct *cwq)
321 spin_unlock_irqrestore(&cwq->lock, flags); 321 spin_unlock_irqrestore(&cwq->lock, flags);
322 322
323 BUG_ON(get_wq_data(work) != cwq); 323 BUG_ON(get_wq_data(work) != cwq);
324 if (!test_bit(WORK_STRUCT_NOAUTOREL, &work->management)) 324 if (!test_bit(WORK_STRUCT_NOAUTOREL, work_data_bits(work)))
325 work_release(work); 325 work_release(work);
326 f(work); 326 f(work);
327 327
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-25 12:28:37 -0500