aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
authorIngo Molnar <mingo@kernel.org>2013-11-06 01:50:37 -0500
committerIngo Molnar <mingo@kernel.org>2013-11-06 01:50:37 -0500
commitc90423d1de12fbeaf0c898e1db0e962de347302b (patch)
tree8c7a32b37e74155324ae2b556fcc42718ccf29a3 /include/linux
parentecf1f014325ba60f4df35edae1a357c67c5d4eb1 (diff)
parentb8a216269ec0ce2e961d32e6d640d7010b8a818e (diff)
Merge branch 'sched/core' into core/locking, to prepare the kernel/locking/ file move
Conflicts: kernel/Makefile There are conflicts in kernel/Makefile due to file moving in the scheduler tree - resolve them. Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/completion.h2
-rw-r--r--include/linux/hardirq.h8
-rw-r--r--include/linux/mempolicy.h1
-rw-r--r--include/linux/migrate.h7
-rw-r--r--include/linux/mm.h118
-rw-r--r--include/linux/mm_types.h17
-rw-r--r--include/linux/page-flags-layout.h28
-rw-r--r--include/linux/preempt.h112
-rw-r--r--include/linux/sched.h167
-rw-r--r--include/linux/sched/sysctl.h1
-rw-r--r--include/linux/stop_machine.h1
-rw-r--r--include/linux/thread_info.h17
-rw-r--r--include/linux/topology.h6
-rw-r--r--include/linux/tty.h28
-rw-r--r--include/linux/uaccess.h8
-rw-r--r--include/linux/wait.h374
16 files changed, 494 insertions, 401 deletions
diff --git a/include/linux/completion.h b/include/linux/completion.h
index 3cd574d5b19e..22c33e35bcb2 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -5,7 +5,7 @@
5 * (C) Copyright 2001 Linus Torvalds 5 * (C) Copyright 2001 Linus Torvalds
6 * 6 *
7 * Atomic wait-for-completion handler data structures. 7 * Atomic wait-for-completion handler data structures.
8 * See kernel/sched/core.c for details. 8 * See kernel/sched/completion.c for details.
9 */ 9 */
10 10
11#include <linux/wait.h> 11#include <linux/wait.h>
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 1e041063b226..d9cf963ac832 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -33,7 +33,7 @@ extern void rcu_nmi_exit(void);
33#define __irq_enter() \ 33#define __irq_enter() \
34 do { \ 34 do { \
35 account_irq_enter_time(current); \ 35 account_irq_enter_time(current); \
36 add_preempt_count(HARDIRQ_OFFSET); \ 36 preempt_count_add(HARDIRQ_OFFSET); \
37 trace_hardirq_enter(); \ 37 trace_hardirq_enter(); \
38 } while (0) 38 } while (0)
39 39
@@ -49,7 +49,7 @@ extern void irq_enter(void);
49 do { \ 49 do { \
50 trace_hardirq_exit(); \ 50 trace_hardirq_exit(); \
51 account_irq_exit_time(current); \ 51 account_irq_exit_time(current); \
52 sub_preempt_count(HARDIRQ_OFFSET); \ 52 preempt_count_sub(HARDIRQ_OFFSET); \
53 } while (0) 53 } while (0)
54 54
55/* 55/*
@@ -62,7 +62,7 @@ extern void irq_exit(void);
62 lockdep_off(); \ 62 lockdep_off(); \
63 ftrace_nmi_enter(); \ 63 ftrace_nmi_enter(); \
64 BUG_ON(in_nmi()); \ 64 BUG_ON(in_nmi()); \
65 add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \ 65 preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
66 rcu_nmi_enter(); \ 66 rcu_nmi_enter(); \
67 trace_hardirq_enter(); \ 67 trace_hardirq_enter(); \
68 } while (0) 68 } while (0)
@@ -72,7 +72,7 @@ extern void irq_exit(void);
72 trace_hardirq_exit(); \ 72 trace_hardirq_exit(); \
73 rcu_nmi_exit(); \ 73 rcu_nmi_exit(); \
74 BUG_ON(!in_nmi()); \ 74 BUG_ON(!in_nmi()); \
75 sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \ 75 preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
76 ftrace_nmi_exit(); \ 76 ftrace_nmi_exit(); \
77 lockdep_on(); \ 77 lockdep_on(); \
78 } while (0) 78 } while (0)
diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h
index da6716b9e3fe..ea4d2495c646 100644
--- a/include/linux/mempolicy.h
+++ b/include/linux/mempolicy.h
@@ -136,6 +136,7 @@ struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
136 136
137struct mempolicy *get_vma_policy(struct task_struct *tsk, 137struct mempolicy *get_vma_policy(struct task_struct *tsk,
138 struct vm_area_struct *vma, unsigned long addr); 138 struct vm_area_struct *vma, unsigned long addr);
139bool vma_policy_mof(struct task_struct *task, struct vm_area_struct *vma);
139 140
140extern void numa_default_policy(void); 141extern void numa_default_policy(void);
141extern void numa_policy_init(void); 142extern void numa_policy_init(void);
diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 8d3c57fdf221..f5096b58b20d 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -90,11 +90,12 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping,
90#endif /* CONFIG_MIGRATION */ 90#endif /* CONFIG_MIGRATION */
91 91
92#ifdef CONFIG_NUMA_BALANCING 92#ifdef CONFIG_NUMA_BALANCING
93extern int migrate_misplaced_page(struct page *page, int node); 93extern int migrate_misplaced_page(struct page *page,
94extern int migrate_misplaced_page(struct page *page, int node); 94 struct vm_area_struct *vma, int node);
95extern bool migrate_ratelimited(int node); 95extern bool migrate_ratelimited(int node);
96#else 96#else
97static inline int migrate_misplaced_page(struct page *page, int node) 97static inline int migrate_misplaced_page(struct page *page,
98 struct vm_area_struct *vma, int node)
98{ 99{
99 return -EAGAIN; /* can't migrate now */ 100 return -EAGAIN; /* can't migrate now */
100} 101}
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8b6e55ee8855..81443d557a2e 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -581,11 +581,11 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
581 * sets it, so none of the operations on it need to be atomic. 581 * sets it, so none of the operations on it need to be atomic.
582 */ 582 */
583 583
584/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NID] | ... | FLAGS | */ 584/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
585#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) 585#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
586#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) 586#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
587#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) 587#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
588#define LAST_NID_PGOFF (ZONES_PGOFF - LAST_NID_WIDTH) 588#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
589 589
590/* 590/*
591 * Define the bit shifts to access each section. For non-existent 591 * Define the bit shifts to access each section. For non-existent
@@ -595,7 +595,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
595#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) 595#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
596#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) 596#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
597#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) 597#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
598#define LAST_NID_PGSHIFT (LAST_NID_PGOFF * (LAST_NID_WIDTH != 0)) 598#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
599 599
600/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ 600/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
601#ifdef NODE_NOT_IN_PAGE_FLAGS 601#ifdef NODE_NOT_IN_PAGE_FLAGS
@@ -617,7 +617,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
617#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) 617#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
618#define NODES_MASK ((1UL << NODES_WIDTH) - 1) 618#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
619#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) 619#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
620#define LAST_NID_MASK ((1UL << LAST_NID_WIDTH) - 1) 620#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_WIDTH) - 1)
621#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) 621#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
622 622
623static inline enum zone_type page_zonenum(const struct page *page) 623static inline enum zone_type page_zonenum(const struct page *page)
@@ -661,51 +661,117 @@ static inline int page_to_nid(const struct page *page)
661#endif 661#endif
662 662
663#ifdef CONFIG_NUMA_BALANCING 663#ifdef CONFIG_NUMA_BALANCING
664#ifdef LAST_NID_NOT_IN_PAGE_FLAGS 664static inline int cpu_pid_to_cpupid(int cpu, int pid)
665static inline int page_nid_xchg_last(struct page *page, int nid)
666{ 665{
667 return xchg(&page->_last_nid, nid); 666 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
668} 667}
669 668
670static inline int page_nid_last(struct page *page) 669static inline int cpupid_to_pid(int cpupid)
671{ 670{
672 return page->_last_nid; 671 return cpupid & LAST__PID_MASK;
673} 672}
674static inline void page_nid_reset_last(struct page *page) 673
674static inline int cpupid_to_cpu(int cpupid)
675{ 675{
676 page->_last_nid = -1; 676 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
677} 677}
678#else 678
679static inline int page_nid_last(struct page *page) 679static inline int cpupid_to_nid(int cpupid)
680{ 680{
681 return (page->flags >> LAST_NID_PGSHIFT) & LAST_NID_MASK; 681 return cpu_to_node(cpupid_to_cpu(cpupid));
682} 682}
683 683
684extern int page_nid_xchg_last(struct page *page, int nid); 684static inline bool cpupid_pid_unset(int cpupid)
685{
686 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
687}
685 688
686static inline void page_nid_reset_last(struct page *page) 689static inline bool cpupid_cpu_unset(int cpupid)
687{ 690{
688 int nid = (1 << LAST_NID_SHIFT) - 1; 691 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
692}
689 693
690 page->flags &= ~(LAST_NID_MASK << LAST_NID_PGSHIFT); 694static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
691 page->flags |= (nid & LAST_NID_MASK) << LAST_NID_PGSHIFT; 695{
696 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
697}
698
699#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
700#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
701static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
702{
703 return xchg(&page->_last_cpupid, cpupid);
704}
705
706static inline int page_cpupid_last(struct page *page)
707{
708 return page->_last_cpupid;
709}
710static inline void page_cpupid_reset_last(struct page *page)
711{
712 page->_last_cpupid = -1;
692} 713}
693#endif /* LAST_NID_NOT_IN_PAGE_FLAGS */
694#else 714#else
695static inline int page_nid_xchg_last(struct page *page, int nid) 715static inline int page_cpupid_last(struct page *page)
696{ 716{
697 return page_to_nid(page); 717 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
698} 718}
699 719
700static inline int page_nid_last(struct page *page) 720extern int page_cpupid_xchg_last(struct page *page, int cpupid);
721
722static inline void page_cpupid_reset_last(struct page *page)
701{ 723{
702 return page_to_nid(page); 724 int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
725
726 page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
727 page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
728}
729#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
730#else /* !CONFIG_NUMA_BALANCING */
731static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
732{
733 return page_to_nid(page); /* XXX */
703} 734}
704 735
705static inline void page_nid_reset_last(struct page *page) 736static inline int page_cpupid_last(struct page *page)
706{ 737{
738 return page_to_nid(page); /* XXX */
707} 739}
708#endif 740
741static inline int cpupid_to_nid(int cpupid)
742{
743 return -1;
744}
745
746static inline int cpupid_to_pid(int cpupid)
747{
748 return -1;
749}
750
751static inline int cpupid_to_cpu(int cpupid)
752{
753 return -1;
754}
755
756static inline int cpu_pid_to_cpupid(int nid, int pid)
757{
758 return -1;
759}
760
761static inline bool cpupid_pid_unset(int cpupid)
762{
763 return 1;
764}
765
766static inline void page_cpupid_reset_last(struct page *page)
767{
768}
769
770static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
771{
772 return false;
773}
774#endif /* CONFIG_NUMA_BALANCING */
709 775
710static inline struct zone *page_zone(const struct page *page) 776static inline struct zone *page_zone(const struct page *page)
711{ 777{
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index d9851eeb6e1d..a3198e5aaf4e 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -174,8 +174,8 @@ struct page {
174 void *shadow; 174 void *shadow;
175#endif 175#endif
176 176
177#ifdef LAST_NID_NOT_IN_PAGE_FLAGS 177#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
178 int _last_nid; 178 int _last_cpupid;
179#endif 179#endif
180} 180}
181/* 181/*
@@ -420,28 +420,15 @@ struct mm_struct {
420 */ 420 */
421 unsigned long numa_next_scan; 421 unsigned long numa_next_scan;
422 422
423 /* numa_next_reset is when the PTE scanner period will be reset */
424 unsigned long numa_next_reset;
425
426 /* Restart point for scanning and setting pte_numa */ 423 /* Restart point for scanning and setting pte_numa */
427 unsigned long numa_scan_offset; 424 unsigned long numa_scan_offset;
428 425
429 /* numa_scan_seq prevents two threads setting pte_numa */ 426 /* numa_scan_seq prevents two threads setting pte_numa */
430 int numa_scan_seq; 427 int numa_scan_seq;
431
432 /*
433 * The first node a task was scheduled on. If a task runs on
434 * a different node than Make PTE Scan Go Now.
435 */
436 int first_nid;
437#endif 428#endif
438 struct uprobes_state uprobes_state; 429 struct uprobes_state uprobes_state;
439}; 430};
440 431
441/* first nid will either be a valid NID or one of these values */
442#define NUMA_PTE_SCAN_INIT -1
443#define NUMA_PTE_SCAN_ACTIVE -2
444
445static inline void mm_init_cpumask(struct mm_struct *mm) 432static inline void mm_init_cpumask(struct mm_struct *mm)
446{ 433{
447#ifdef CONFIG_CPUMASK_OFFSTACK 434#ifdef CONFIG_CPUMASK_OFFSTACK
diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h
index 93506a114034..da523661500a 100644
--- a/include/linux/page-flags-layout.h
+++ b/include/linux/page-flags-layout.h
@@ -38,10 +38,10 @@
38 * The last is when there is insufficient space in page->flags and a separate 38 * The last is when there is insufficient space in page->flags and a separate
39 * lookup is necessary. 39 * lookup is necessary.
40 * 40 *
41 * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS | 41 * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
42 * " plus space for last_nid: | NODE | ZONE | LAST_NID ... | FLAGS | 42 * " plus space for last_cpupid: | NODE | ZONE | LAST_CPUPID ... | FLAGS |
43 * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS | 43 * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
44 * " plus space for last_nid: | SECTION | NODE | ZONE | LAST_NID ... | FLAGS | 44 * " plus space for last_cpupid: | SECTION | NODE | ZONE | LAST_CPUPID ... | FLAGS |
45 * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS | 45 * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
46 */ 46 */
47#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 47#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
@@ -62,15 +62,21 @@
62#endif 62#endif
63 63
64#ifdef CONFIG_NUMA_BALANCING 64#ifdef CONFIG_NUMA_BALANCING
65#define LAST_NID_SHIFT NODES_SHIFT 65#define LAST__PID_SHIFT 8
66#define LAST__PID_MASK ((1 << LAST__PID_SHIFT)-1)
67
68#define LAST__CPU_SHIFT NR_CPUS_BITS
69#define LAST__CPU_MASK ((1 << LAST__CPU_SHIFT)-1)
70
71#define LAST_CPUPID_SHIFT (LAST__PID_SHIFT+LAST__CPU_SHIFT)
66#else 72#else
67#define LAST_NID_SHIFT 0 73#define LAST_CPUPID_SHIFT 0
68#endif 74#endif
69 75
70#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_NID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS 76#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
71#define LAST_NID_WIDTH LAST_NID_SHIFT 77#define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT
72#else 78#else
73#define LAST_NID_WIDTH 0 79#define LAST_CPUPID_WIDTH 0
74#endif 80#endif
75 81
76/* 82/*
@@ -81,8 +87,8 @@
81#define NODE_NOT_IN_PAGE_FLAGS 87#define NODE_NOT_IN_PAGE_FLAGS
82#endif 88#endif
83 89
84#if defined(CONFIG_NUMA_BALANCING) && LAST_NID_WIDTH == 0 90#if defined(CONFIG_NUMA_BALANCING) && LAST_CPUPID_WIDTH == 0
85#define LAST_NID_NOT_IN_PAGE_FLAGS 91#define LAST_CPUPID_NOT_IN_PAGE_FLAGS
86#endif 92#endif
87 93
88#endif /* _LINUX_PAGE_FLAGS_LAYOUT */ 94#endif /* _LINUX_PAGE_FLAGS_LAYOUT */
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index f5d4723cdb3d..a3d9dc8c2c00 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -6,106 +6,95 @@
6 * preempt_count (used for kernel preemption, interrupt count, etc.) 6 * preempt_count (used for kernel preemption, interrupt count, etc.)
7 */ 7 */
8 8
9#include <linux/thread_info.h>
10#include <linux/linkage.h> 9#include <linux/linkage.h>
11#include <linux/list.h> 10#include <linux/list.h>
12 11
13#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER) 12/*
14 extern void add_preempt_count(int val); 13 * We use the MSB mostly because its available; see <linux/preempt_mask.h> for
15 extern void sub_preempt_count(int val); 14 * the other bits -- can't include that header due to inclusion hell.
16#else 15 */
17# define add_preempt_count(val) do { preempt_count() += (val); } while (0) 16#define PREEMPT_NEED_RESCHED 0x80000000
18# define sub_preempt_count(val) do { preempt_count() -= (val); } while (0)
19#endif
20
21#define inc_preempt_count() add_preempt_count(1)
22#define dec_preempt_count() sub_preempt_count(1)
23
24#define preempt_count() (current_thread_info()->preempt_count)
25
26#ifdef CONFIG_PREEMPT
27
28asmlinkage void preempt_schedule(void);
29
30#define preempt_check_resched() \
31do { \
32 if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
33 preempt_schedule(); \
34} while (0)
35
36#ifdef CONFIG_CONTEXT_TRACKING
37 17
38void preempt_schedule_context(void); 18#include <asm/preempt.h>
39 19
40#define preempt_check_resched_context() \ 20#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
41do { \ 21extern void preempt_count_add(int val);
42 if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ 22extern void preempt_count_sub(int val);
43 preempt_schedule_context(); \ 23#define preempt_count_dec_and_test() ({ preempt_count_sub(1); should_resched(); })
44} while (0)
45#else 24#else
25#define preempt_count_add(val) __preempt_count_add(val)
26#define preempt_count_sub(val) __preempt_count_sub(val)
27#define preempt_count_dec_and_test() __preempt_count_dec_and_test()
28#endif
46 29
47#define preempt_check_resched_context() preempt_check_resched() 30#define __preempt_count_inc() __preempt_count_add(1)
48 31#define __preempt_count_dec() __preempt_count_sub(1)
49#endif /* CONFIG_CONTEXT_TRACKING */
50
51#else /* !CONFIG_PREEMPT */
52
53#define preempt_check_resched() do { } while (0)
54#define preempt_check_resched_context() do { } while (0)
55
56#endif /* CONFIG_PREEMPT */
57 32
33#define preempt_count_inc() preempt_count_add(1)
34#define preempt_count_dec() preempt_count_sub(1)
58 35
59#ifdef CONFIG_PREEMPT_COUNT 36#ifdef CONFIG_PREEMPT_COUNT
60 37
61#define preempt_disable() \ 38#define preempt_disable() \
62do { \ 39do { \
63 inc_preempt_count(); \ 40 preempt_count_inc(); \
64 barrier(); \ 41 barrier(); \
65} while (0) 42} while (0)
66 43
67#define sched_preempt_enable_no_resched() \ 44#define sched_preempt_enable_no_resched() \
68do { \ 45do { \
69 barrier(); \ 46 barrier(); \
70 dec_preempt_count(); \ 47 preempt_count_dec(); \
71} while (0) 48} while (0)
72 49
73#define preempt_enable_no_resched() sched_preempt_enable_no_resched() 50#define preempt_enable_no_resched() sched_preempt_enable_no_resched()
74 51
52#ifdef CONFIG_PREEMPT
75#define preempt_enable() \ 53#define preempt_enable() \
76do { \ 54do { \
77 preempt_enable_no_resched(); \
78 barrier(); \ 55 barrier(); \
79 preempt_check_resched(); \ 56 if (unlikely(preempt_count_dec_and_test())) \
57 __preempt_schedule(); \
58} while (0)
59
60#define preempt_check_resched() \
61do { \
62 if (should_resched()) \
63 __preempt_schedule(); \
80} while (0) 64} while (0)
81 65
82/* For debugging and tracer internals only! */ 66#else
83#define add_preempt_count_notrace(val) \ 67#define preempt_enable() preempt_enable_no_resched()
84 do { preempt_count() += (val); } while (0) 68#define preempt_check_resched() do { } while (0)
85#define sub_preempt_count_notrace(val) \ 69#endif
86 do { preempt_count() -= (val); } while (0)
87#define inc_preempt_count_notrace() add_preempt_count_notrace(1)
88#define dec_preempt_count_notrace() sub_preempt_count_notrace(1)
89 70
90#define preempt_disable_notrace() \ 71#define preempt_disable_notrace() \
91do { \ 72do { \
92 inc_preempt_count_notrace(); \ 73 __preempt_count_inc(); \
93 barrier(); \ 74 barrier(); \
94} while (0) 75} while (0)
95 76
96#define preempt_enable_no_resched_notrace() \ 77#define preempt_enable_no_resched_notrace() \
97do { \ 78do { \
98 barrier(); \ 79 barrier(); \
99 dec_preempt_count_notrace(); \ 80 __preempt_count_dec(); \
100} while (0) 81} while (0)
101 82
102/* preempt_check_resched is OK to trace */ 83#ifdef CONFIG_PREEMPT
84
85#ifndef CONFIG_CONTEXT_TRACKING
86#define __preempt_schedule_context() __preempt_schedule()
87#endif
88
103#define preempt_enable_notrace() \ 89#define preempt_enable_notrace() \
104do { \ 90do { \
105 preempt_enable_no_resched_notrace(); \
106 barrier(); \ 91 barrier(); \
107 preempt_check_resched_context(); \ 92 if (unlikely(__preempt_count_dec_and_test())) \
93 __preempt_schedule_context(); \
108} while (0) 94} while (0)
95#else
96#define preempt_enable_notrace() preempt_enable_no_resched_notrace()
97#endif
109 98
110#else /* !CONFIG_PREEMPT_COUNT */ 99#else /* !CONFIG_PREEMPT_COUNT */
111 100
@@ -115,10 +104,11 @@ do { \
115 * that can cause faults and scheduling migrate into our preempt-protected 104 * that can cause faults and scheduling migrate into our preempt-protected
116 * region. 105 * region.
117 */ 106 */
118#define preempt_disable() barrier() 107#define preempt_disable() barrier()
119#define sched_preempt_enable_no_resched() barrier() 108#define sched_preempt_enable_no_resched() barrier()
120#define preempt_enable_no_resched() barrier() 109#define preempt_enable_no_resched() barrier()
121#define preempt_enable() barrier() 110#define preempt_enable() barrier()
111#define preempt_check_resched() do { } while (0)
122 112
123#define preempt_disable_notrace() barrier() 113#define preempt_disable_notrace() barrier()
124#define preempt_enable_no_resched_notrace() barrier() 114#define preempt_enable_no_resched_notrace() barrier()
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e27baeeda3f4..045b0d227846 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -22,6 +22,7 @@ struct sched_param {
22#include <linux/errno.h> 22#include <linux/errno.h>
23#include <linux/nodemask.h> 23#include <linux/nodemask.h>
24#include <linux/mm_types.h> 24#include <linux/mm_types.h>
25#include <linux/preempt.h>
25 26
26#include <asm/page.h> 27#include <asm/page.h>
27#include <asm/ptrace.h> 28#include <asm/ptrace.h>
@@ -427,6 +428,14 @@ struct task_cputime {
427 .sum_exec_runtime = 0, \ 428 .sum_exec_runtime = 0, \
428 } 429 }
429 430
431#define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED)
432
433#ifdef CONFIG_PREEMPT_COUNT
434#define PREEMPT_DISABLED (1 + PREEMPT_ENABLED)
435#else
436#define PREEMPT_DISABLED PREEMPT_ENABLED
437#endif
438
430/* 439/*
431 * Disable preemption until the scheduler is running. 440 * Disable preemption until the scheduler is running.
432 * Reset by start_kernel()->sched_init()->init_idle(). 441 * Reset by start_kernel()->sched_init()->init_idle().
@@ -434,7 +443,7 @@ struct task_cputime {
434 * We include PREEMPT_ACTIVE to avoid cond_resched() from working 443 * We include PREEMPT_ACTIVE to avoid cond_resched() from working
435 * before the scheduler is active -- see should_resched(). 444 * before the scheduler is active -- see should_resched().
436 */ 445 */
437#define INIT_PREEMPT_COUNT (1 + PREEMPT_ACTIVE) 446#define INIT_PREEMPT_COUNT (PREEMPT_DISABLED + PREEMPT_ACTIVE)
438 447
439/** 448/**
440 * struct thread_group_cputimer - thread group interval timer counts 449 * struct thread_group_cputimer - thread group interval timer counts
@@ -768,6 +777,7 @@ enum cpu_idle_type {
768#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ 777#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */
769#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ 778#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */
770#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ 779#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */
780#define SD_NUMA 0x4000 /* cross-node balancing */
771 781
772extern int __weak arch_sd_sibiling_asym_packing(void); 782extern int __weak arch_sd_sibiling_asym_packing(void);
773 783
@@ -811,6 +821,10 @@ struct sched_domain {
811 821
812 u64 last_update; 822 u64 last_update;
813 823
824 /* idle_balance() stats */
825 u64 max_newidle_lb_cost;
826 unsigned long next_decay_max_lb_cost;
827
814#ifdef CONFIG_SCHEDSTATS 828#ifdef CONFIG_SCHEDSTATS
815 /* load_balance() stats */ 829 /* load_balance() stats */
816 unsigned int lb_count[CPU_MAX_IDLE_TYPES]; 830 unsigned int lb_count[CPU_MAX_IDLE_TYPES];
@@ -1029,6 +1043,8 @@ struct task_struct {
1029 struct task_struct *last_wakee; 1043 struct task_struct *last_wakee;
1030 unsigned long wakee_flips; 1044 unsigned long wakee_flips;
1031 unsigned long wakee_flip_decay_ts; 1045 unsigned long wakee_flip_decay_ts;
1046
1047 int wake_cpu;
1032#endif 1048#endif
1033 int on_rq; 1049 int on_rq;
1034 1050
@@ -1324,10 +1340,41 @@ struct task_struct {
1324#endif 1340#endif
1325#ifdef CONFIG_NUMA_BALANCING 1341#ifdef CONFIG_NUMA_BALANCING
1326 int numa_scan_seq; 1342 int numa_scan_seq;
1327 int numa_migrate_seq;
1328 unsigned int numa_scan_period; 1343 unsigned int numa_scan_period;
1344 unsigned int numa_scan_period_max;
1345 int numa_preferred_nid;
1346 int numa_migrate_deferred;
1347 unsigned long numa_migrate_retry;
1329 u64 node_stamp; /* migration stamp */ 1348 u64 node_stamp; /* migration stamp */
1330 struct callback_head numa_work; 1349 struct callback_head numa_work;
1350
1351 struct list_head numa_entry;
1352 struct numa_group *numa_group;
1353
1354 /*
1355 * Exponential decaying average of faults on a per-node basis.
1356 * Scheduling placement decisions are made based on the these counts.
1357 * The values remain static for the duration of a PTE scan
1358 */
1359 unsigned long *numa_faults;
1360 unsigned long total_numa_faults;
1361
1362 /*
1363 * numa_faults_buffer records faults per node during the current
1364 * scan window. When the scan completes, the counts in numa_faults
1365 * decay and these values are copied.
1366 */
1367 unsigned long *numa_faults_buffer;
1368
1369 /*
1370 * numa_faults_locality tracks if faults recorded during the last
1371 * scan window were remote/local. The task scan period is adapted
1372 * based on the locality of the faults with different weights
1373 * depending on whether they were shared or private faults
1374 */
1375 unsigned long numa_faults_locality[2];
1376
1377 unsigned long numa_pages_migrated;
1331#endif /* CONFIG_NUMA_BALANCING */ 1378#endif /* CONFIG_NUMA_BALANCING */
1332 1379
1333 struct rcu_head rcu; 1380 struct rcu_head rcu;
@@ -1412,16 +1459,33 @@ struct task_struct {
1412/* Future-safe accessor for struct task_struct's cpus_allowed. */ 1459/* Future-safe accessor for struct task_struct's cpus_allowed. */
1413#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) 1460#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
1414 1461
1462#define TNF_MIGRATED 0x01
1463#define TNF_NO_GROUP 0x02
1464#define TNF_SHARED 0x04
1465#define TNF_FAULT_LOCAL 0x08
1466
1415#ifdef CONFIG_NUMA_BALANCING 1467#ifdef CONFIG_NUMA_BALANCING
1416extern void task_numa_fault(int node, int pages, bool migrated); 1468extern void task_numa_fault(int last_node, int node, int pages, int flags);
1469extern pid_t task_numa_group_id(struct task_struct *p);
1417extern void set_numabalancing_state(bool enabled); 1470extern void set_numabalancing_state(bool enabled);
1471extern void task_numa_free(struct task_struct *p);
1472
1473extern unsigned int sysctl_numa_balancing_migrate_deferred;
1418#else 1474#else
1419static inline void task_numa_fault(int node, int pages, bool migrated) 1475static inline void task_numa_fault(int last_node, int node, int pages,
1476 int flags)
1420{ 1477{
1421} 1478}
1479static inline pid_t task_numa_group_id(struct task_struct *p)
1480{
1481 return 0;
1482}
1422static inline void set_numabalancing_state(bool enabled) 1483static inline void set_numabalancing_state(bool enabled)
1423{ 1484{
1424} 1485}
1486static inline void task_numa_free(struct task_struct *p)
1487{
1488}
1425#endif 1489#endif
1426 1490
1427static inline struct pid *task_pid(struct task_struct *task) 1491static inline struct pid *task_pid(struct task_struct *task)
@@ -1974,7 +2038,7 @@ extern void wake_up_new_task(struct task_struct *tsk);
1974#else 2038#else
1975 static inline void kick_process(struct task_struct *tsk) { } 2039 static inline void kick_process(struct task_struct *tsk) { }
1976#endif 2040#endif
1977extern void sched_fork(struct task_struct *p); 2041extern void sched_fork(unsigned long clone_flags, struct task_struct *p);
1978extern void sched_dead(struct task_struct *p); 2042extern void sched_dead(struct task_struct *p);
1979 2043
1980extern void proc_caches_init(void); 2044extern void proc_caches_init(void);
@@ -2401,11 +2465,6 @@ static inline int signal_pending_state(long state, struct task_struct *p)
2401 return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); 2465 return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
2402} 2466}
2403 2467
2404static inline int need_resched(void)
2405{
2406 return unlikely(test_thread_flag(TIF_NEED_RESCHED));
2407}
2408
2409/* 2468/*
2410 * cond_resched() and cond_resched_lock(): latency reduction via 2469 * cond_resched() and cond_resched_lock(): latency reduction via
2411 * explicit rescheduling in places that are safe. The return 2470 * explicit rescheduling in places that are safe. The return
@@ -2474,36 +2533,105 @@ static inline int tsk_is_polling(struct task_struct *p)
2474{ 2533{
2475 return task_thread_info(p)->status & TS_POLLING; 2534 return task_thread_info(p)->status & TS_POLLING;
2476} 2535}
2477static inline void current_set_polling(void) 2536static inline void __current_set_polling(void)
2478{ 2537{
2479 current_thread_info()->status |= TS_POLLING; 2538 current_thread_info()->status |= TS_POLLING;
2480} 2539}
2481 2540
2482static inline void current_clr_polling(void) 2541static inline bool __must_check current_set_polling_and_test(void)
2542{
2543 __current_set_polling();
2544
2545 /*
2546 * Polling state must be visible before we test NEED_RESCHED,
2547 * paired by resched_task()
2548 */
2549 smp_mb();
2550
2551 return unlikely(tif_need_resched());
2552}
2553
2554static inline void __current_clr_polling(void)
2483{ 2555{
2484 current_thread_info()->status &= ~TS_POLLING; 2556 current_thread_info()->status &= ~TS_POLLING;
2485 smp_mb__after_clear_bit(); 2557}
2558
2559static inline bool __must_check current_clr_polling_and_test(void)
2560{
2561 __current_clr_polling();
2562
2563 /*
2564 * Polling state must be visible before we test NEED_RESCHED,
2565 * paired by resched_task()
2566 */
2567 smp_mb();
2568
2569 return unlikely(tif_need_resched());
2486} 2570}
2487#elif defined(TIF_POLLING_NRFLAG) 2571#elif defined(TIF_POLLING_NRFLAG)
2488static inline int tsk_is_polling(struct task_struct *p) 2572static inline int tsk_is_polling(struct task_struct *p)
2489{ 2573{
2490 return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG); 2574 return test_tsk_thread_flag(p, TIF_POLLING_NRFLAG);
2491} 2575}
2492static inline void current_set_polling(void) 2576
2577static inline void __current_set_polling(void)
2493{ 2578{
2494 set_thread_flag(TIF_POLLING_NRFLAG); 2579 set_thread_flag(TIF_POLLING_NRFLAG);
2495} 2580}
2496 2581
2497static inline void current_clr_polling(void) 2582static inline bool __must_check current_set_polling_and_test(void)
2583{
2584 __current_set_polling();
2585
2586 /*
2587 * Polling state must be visible before we test NEED_RESCHED,
2588 * paired by resched_task()
2589 *
2590 * XXX: assumes set/clear bit are identical barrier wise.
2591 */
2592 smp_mb__after_clear_bit();
2593
2594 return unlikely(tif_need_resched());
2595}
2596
2597static inline void __current_clr_polling(void)
2498{ 2598{
2499 clear_thread_flag(TIF_POLLING_NRFLAG); 2599 clear_thread_flag(TIF_POLLING_NRFLAG);
2500} 2600}
2601
2602static inline bool __must_check current_clr_polling_and_test(void)
2603{
2604 __current_clr_polling();
2605
2606 /*
2607 * Polling state must be visible before we test NEED_RESCHED,
2608 * paired by resched_task()
2609 */
2610 smp_mb__after_clear_bit();
2611
2612 return unlikely(tif_need_resched());
2613}
2614
2501#else 2615#else
2502static inline int tsk_is_polling(struct task_struct *p) { return 0; } 2616static inline int tsk_is_polling(struct task_struct *p) { return 0; }
2503static inline void current_set_polling(void) { } 2617static inline void __current_set_polling(void) { }
2504static inline void current_clr_polling(void) { } 2618static inline void __current_clr_polling(void) { }
2619
2620static inline bool __must_check current_set_polling_and_test(void)
2621{
2622 return unlikely(tif_need_resched());
2623}
2624static inline bool __must_check current_clr_polling_and_test(void)
2625{
2626 return unlikely(tif_need_resched());
2627}
2505#endif 2628#endif
2506 2629
2630static __always_inline bool need_resched(void)
2631{
2632 return unlikely(tif_need_resched());
2633}
2634
2507/* 2635/*
2508 * Thread group CPU time accounting. 2636 * Thread group CPU time accounting.
2509 */ 2637 */
@@ -2545,6 +2673,11 @@ static inline unsigned int task_cpu(const struct task_struct *p)
2545 return task_thread_info(p)->cpu; 2673 return task_thread_info(p)->cpu;
2546} 2674}
2547 2675
2676static inline int task_node(const struct task_struct *p)
2677{
2678 return cpu_to_node(task_cpu(p));
2679}
2680
2548extern void set_task_cpu(struct task_struct *p, unsigned int cpu); 2681extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
2549 2682
2550#else 2683#else
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index 9552afa733d8..41467f8ff8ec 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -47,7 +47,6 @@ extern enum sched_tunable_scaling sysctl_sched_tunable_scaling;
47extern unsigned int sysctl_numa_balancing_scan_delay; 47extern unsigned int sysctl_numa_balancing_scan_delay;
48extern unsigned int sysctl_numa_balancing_scan_period_min; 48extern unsigned int sysctl_numa_balancing_scan_period_min;
49extern unsigned int sysctl_numa_balancing_scan_period_max; 49extern unsigned int sysctl_numa_balancing_scan_period_max;
50extern unsigned int sysctl_numa_balancing_scan_period_reset;
51extern unsigned int sysctl_numa_balancing_scan_size; 50extern unsigned int sysctl_numa_balancing_scan_size;
52extern unsigned int sysctl_numa_balancing_settle_count; 51extern unsigned int sysctl_numa_balancing_settle_count;
53 52
diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h
index 3b5e910d14ca..d2abbdb8c6aa 100644
--- a/include/linux/stop_machine.h
+++ b/include/linux/stop_machine.h
@@ -28,6 +28,7 @@ struct cpu_stop_work {
28}; 28};
29 29
30int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg); 30int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg);
31int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg);
31void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, 32void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
32 struct cpu_stop_work *work_buf); 33 struct cpu_stop_work *work_buf);
33int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg); 34int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg);
diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h
index e7e04736802f..fddbe2023a5d 100644
--- a/include/linux/thread_info.h
+++ b/include/linux/thread_info.h
@@ -104,8 +104,21 @@ static inline int test_ti_thread_flag(struct thread_info *ti, int flag)
104#define test_thread_flag(flag) \ 104#define test_thread_flag(flag) \
105 test_ti_thread_flag(current_thread_info(), flag) 105 test_ti_thread_flag(current_thread_info(), flag)
106 106
107#define set_need_resched() set_thread_flag(TIF_NEED_RESCHED) 107static inline __deprecated void set_need_resched(void)
108#define clear_need_resched() clear_thread_flag(TIF_NEED_RESCHED) 108{
109 /*
110 * Use of this function in deprecated.
111 *
112 * As of this writing there are only a few users in the DRM tree left
113 * all of which are wrong and can be removed without causing too much
114 * grief.
115 *
116 * The DRM people are aware and are working on removing the last few
117 * instances.
118 */
119}
120
121#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED)
109 122
110#if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK 123#if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK
111/* 124/*
diff --git a/include/linux/topology.h b/include/linux/topology.h
index d3cf0d6e7712..12ae6ce997d6 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -106,6 +106,8 @@ int arch_update_cpu_topology(void);
106 .last_balance = jiffies, \ 106 .last_balance = jiffies, \
107 .balance_interval = 1, \ 107 .balance_interval = 1, \
108 .smt_gain = 1178, /* 15% */ \ 108 .smt_gain = 1178, /* 15% */ \
109 .max_newidle_lb_cost = 0, \
110 .next_decay_max_lb_cost = jiffies, \
109} 111}
110#endif 112#endif
111#endif /* CONFIG_SCHED_SMT */ 113#endif /* CONFIG_SCHED_SMT */
@@ -135,6 +137,8 @@ int arch_update_cpu_topology(void);
135 , \ 137 , \
136 .last_balance = jiffies, \ 138 .last_balance = jiffies, \
137 .balance_interval = 1, \ 139 .balance_interval = 1, \
140 .max_newidle_lb_cost = 0, \
141 .next_decay_max_lb_cost = jiffies, \
138} 142}
139#endif 143#endif
140#endif /* CONFIG_SCHED_MC */ 144#endif /* CONFIG_SCHED_MC */
@@ -166,6 +170,8 @@ int arch_update_cpu_topology(void);
166 , \ 170 , \
167 .last_balance = jiffies, \ 171 .last_balance = jiffies, \
168 .balance_interval = 1, \ 172 .balance_interval = 1, \
173 .max_newidle_lb_cost = 0, \
174 .next_decay_max_lb_cost = jiffies, \
169} 175}
170#endif 176#endif
171 177
diff --git a/include/linux/tty.h b/include/linux/tty.h
index 64f864651d86..633cac77f9f9 100644
--- a/include/linux/tty.h
+++ b/include/linux/tty.h
@@ -672,31 +672,17 @@ static inline void tty_wait_until_sent_from_close(struct tty_struct *tty,
672#define wait_event_interruptible_tty(tty, wq, condition) \ 672#define wait_event_interruptible_tty(tty, wq, condition) \
673({ \ 673({ \
674 int __ret = 0; \ 674 int __ret = 0; \
675 if (!(condition)) { \ 675 if (!(condition)) \
676 __wait_event_interruptible_tty(tty, wq, condition, __ret); \ 676 __ret = __wait_event_interruptible_tty(tty, wq, \
677 } \ 677 condition); \
678 __ret; \ 678 __ret; \
679}) 679})
680 680
681#define __wait_event_interruptible_tty(tty, wq, condition, ret) \ 681#define __wait_event_interruptible_tty(tty, wq, condition) \
682do { \ 682 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
683 DEFINE_WAIT(__wait); \ 683 tty_unlock(tty); \
684 \
685 for (;;) { \
686 prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \
687 if (condition) \
688 break; \
689 if (!signal_pending(current)) { \
690 tty_unlock(tty); \
691 schedule(); \ 684 schedule(); \
692 tty_lock(tty); \ 685 tty_lock(tty))
693 continue; \
694 } \
695 ret = -ERESTARTSYS; \
696 break; \
697 } \
698 finish_wait(&wq, &__wait); \
699} while (0)
700 686
701#ifdef CONFIG_PROC_FS 687#ifdef CONFIG_PROC_FS
702extern void proc_tty_register_driver(struct tty_driver *); 688extern void proc_tty_register_driver(struct tty_driver *);
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 5ca0951e1855..9d8cf056e661 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -15,7 +15,7 @@
15 */ 15 */
16static inline void pagefault_disable(void) 16static inline void pagefault_disable(void)
17{ 17{
18 inc_preempt_count(); 18 preempt_count_inc();
19 /* 19 /*
20 * make sure to have issued the store before a pagefault 20 * make sure to have issued the store before a pagefault
21 * can hit. 21 * can hit.
@@ -30,11 +30,7 @@ static inline void pagefault_enable(void)
30 * the pagefault handler again. 30 * the pagefault handler again.
31 */ 31 */
32 barrier(); 32 barrier();
33 dec_preempt_count(); 33 preempt_count_dec();
34 /*
35 * make sure we do..
36 */
37 barrier();
38 preempt_check_resched(); 34 preempt_check_resched();
39} 35}
40 36
diff --git a/include/linux/wait.h b/include/linux/wait.h
index a67fc1635592..61939ba30aa0 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -1,7 +1,8 @@
1#ifndef _LINUX_WAIT_H 1#ifndef _LINUX_WAIT_H
2#define _LINUX_WAIT_H 2#define _LINUX_WAIT_H
3 3/*
4 4 * Linux wait queue related types and methods
5 */
5#include <linux/list.h> 6#include <linux/list.h>
6#include <linux/stddef.h> 7#include <linux/stddef.h>
7#include <linux/spinlock.h> 8#include <linux/spinlock.h>
@@ -13,27 +14,27 @@ typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, v
13int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key); 14int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key);
14 15
15struct __wait_queue { 16struct __wait_queue {
16 unsigned int flags; 17 unsigned int flags;
17#define WQ_FLAG_EXCLUSIVE 0x01 18#define WQ_FLAG_EXCLUSIVE 0x01
18 void *private; 19 void *private;
19 wait_queue_func_t func; 20 wait_queue_func_t func;
20 struct list_head task_list; 21 struct list_head task_list;
21}; 22};
22 23
23struct wait_bit_key { 24struct wait_bit_key {
24 void *flags; 25 void *flags;
25 int bit_nr; 26 int bit_nr;
26#define WAIT_ATOMIC_T_BIT_NR -1 27#define WAIT_ATOMIC_T_BIT_NR -1
27}; 28};
28 29
29struct wait_bit_queue { 30struct wait_bit_queue {
30 struct wait_bit_key key; 31 struct wait_bit_key key;
31 wait_queue_t wait; 32 wait_queue_t wait;
32}; 33};
33 34
34struct __wait_queue_head { 35struct __wait_queue_head {
35 spinlock_t lock; 36 spinlock_t lock;
36 struct list_head task_list; 37 struct list_head task_list;
37}; 38};
38typedef struct __wait_queue_head wait_queue_head_t; 39typedef struct __wait_queue_head wait_queue_head_t;
39 40
@@ -84,17 +85,17 @@ extern void __init_waitqueue_head(wait_queue_head_t *q, const char *name, struct
84 85
85static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p) 86static inline void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p)
86{ 87{
87 q->flags = 0; 88 q->flags = 0;
88 q->private = p; 89 q->private = p;
89 q->func = default_wake_function; 90 q->func = default_wake_function;
90} 91}
91 92
92static inline void init_waitqueue_func_entry(wait_queue_t *q, 93static inline void
93 wait_queue_func_t func) 94init_waitqueue_func_entry(wait_queue_t *q, wait_queue_func_t func)
94{ 95{
95 q->flags = 0; 96 q->flags = 0;
96 q->private = NULL; 97 q->private = NULL;
97 q->func = func; 98 q->func = func;
98} 99}
99 100
100static inline int waitqueue_active(wait_queue_head_t *q) 101static inline int waitqueue_active(wait_queue_head_t *q)
@@ -114,8 +115,8 @@ static inline void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new)
114/* 115/*
115 * Used for wake-one threads: 116 * Used for wake-one threads:
116 */ 117 */
117static inline void __add_wait_queue_exclusive(wait_queue_head_t *q, 118static inline void
118 wait_queue_t *wait) 119__add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
119{ 120{
120 wait->flags |= WQ_FLAG_EXCLUSIVE; 121 wait->flags |= WQ_FLAG_EXCLUSIVE;
121 __add_wait_queue(q, wait); 122 __add_wait_queue(q, wait);
@@ -127,23 +128,22 @@ static inline void __add_wait_queue_tail(wait_queue_head_t *head,
127 list_add_tail(&new->task_list, &head->task_list); 128 list_add_tail(&new->task_list, &head->task_list);
128} 129}
129 130
130static inline void __add_wait_queue_tail_exclusive(wait_queue_head_t *q, 131static inline void
131 wait_queue_t *wait) 132__add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait)
132{ 133{
133 wait->flags |= WQ_FLAG_EXCLUSIVE; 134 wait->flags |= WQ_FLAG_EXCLUSIVE;
134 __add_wait_queue_tail(q, wait); 135 __add_wait_queue_tail(q, wait);
135} 136}
136 137
137static inline void __remove_wait_queue(wait_queue_head_t *head, 138static inline void
138 wait_queue_t *old) 139__remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old)
139{ 140{
140 list_del(&old->task_list); 141 list_del(&old->task_list);
141} 142}
142 143
143void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key); 144void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
144void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key); 145void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key);
145void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, 146void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key);
146 void *key);
147void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr); 147void __wake_up_locked(wait_queue_head_t *q, unsigned int mode, int nr);
148void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr); 148void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr);
149void __wake_up_bit(wait_queue_head_t *, void *, int); 149void __wake_up_bit(wait_queue_head_t *, void *, int);
@@ -170,27 +170,64 @@ wait_queue_head_t *bit_waitqueue(void *, int);
170/* 170/*
171 * Wakeup macros to be used to report events to the targets. 171 * Wakeup macros to be used to report events to the targets.
172 */ 172 */
173#define wake_up_poll(x, m) \ 173#define wake_up_poll(x, m) \
174 __wake_up(x, TASK_NORMAL, 1, (void *) (m)) 174 __wake_up(x, TASK_NORMAL, 1, (void *) (m))
175#define wake_up_locked_poll(x, m) \ 175#define wake_up_locked_poll(x, m) \
176 __wake_up_locked_key((x), TASK_NORMAL, (void *) (m)) 176 __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
177#define wake_up_interruptible_poll(x, m) \ 177#define wake_up_interruptible_poll(x, m) \
178 __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m)) 178 __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
179#define wake_up_interruptible_sync_poll(x, m) \ 179#define wake_up_interruptible_sync_poll(x, m) \
180 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m)) 180 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
181 181
182#define __wait_event(wq, condition) \ 182#define ___wait_cond_timeout(condition) \
183do { \ 183({ \
184 DEFINE_WAIT(__wait); \ 184 bool __cond = (condition); \
185 if (__cond && !__ret) \
186 __ret = 1; \
187 __cond || !__ret; \
188})
189
190#define ___wait_is_interruptible(state) \
191 (!__builtin_constant_p(state) || \
192 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
193
194#define ___wait_event(wq, condition, state, exclusive, ret, cmd) \
195({ \
196 __label__ __out; \
197 wait_queue_t __wait; \
198 long __ret = ret; \
199 \
200 INIT_LIST_HEAD(&__wait.task_list); \
201 if (exclusive) \
202 __wait.flags = WQ_FLAG_EXCLUSIVE; \
203 else \
204 __wait.flags = 0; \
185 \ 205 \
186 for (;;) { \ 206 for (;;) { \
187 prepare_to_wait(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ 207 long __int = prepare_to_wait_event(&wq, &__wait, state);\
208 \
188 if (condition) \ 209 if (condition) \
189 break; \ 210 break; \
190 schedule(); \ 211 \
212 if (___wait_is_interruptible(state) && __int) { \
213 __ret = __int; \
214 if (exclusive) { \
215 abort_exclusive_wait(&wq, &__wait, \
216 state, NULL); \
217 goto __out; \
218 } \
219 break; \
220 } \
221 \
222 cmd; \
191 } \ 223 } \
192 finish_wait(&wq, &__wait); \ 224 finish_wait(&wq, &__wait); \
193} while (0) 225__out: __ret; \
226})
227
228#define __wait_event(wq, condition) \
229 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
230 schedule())
194 231
195/** 232/**
196 * wait_event - sleep until a condition gets true 233 * wait_event - sleep until a condition gets true
@@ -204,29 +241,17 @@ do { \
204 * wake_up() has to be called after changing any variable that could 241 * wake_up() has to be called after changing any variable that could
205 * change the result of the wait condition. 242 * change the result of the wait condition.
206 */ 243 */
207#define wait_event(wq, condition) \ 244#define wait_event(wq, condition) \
208do { \ 245do { \
209 if (condition) \ 246 if (condition) \
210 break; \ 247 break; \
211 __wait_event(wq, condition); \ 248 __wait_event(wq, condition); \
212} while (0) 249} while (0)
213 250
214#define __wait_event_timeout(wq, condition, ret) \ 251#define __wait_event_timeout(wq, condition, timeout) \
215do { \ 252 ___wait_event(wq, ___wait_cond_timeout(condition), \
216 DEFINE_WAIT(__wait); \ 253 TASK_UNINTERRUPTIBLE, 0, timeout, \
217 \ 254 __ret = schedule_timeout(__ret))
218 for (;;) { \
219 prepare_to_wait(&wq, &__wait, TASK_UNINTERRUPTIBLE); \
220 if (condition) \
221 break; \
222 ret = schedule_timeout(ret); \
223 if (!ret) \
224 break; \
225 } \
226 if (!ret && (condition)) \
227 ret = 1; \
228 finish_wait(&wq, &__wait); \
229} while (0)
230 255
231/** 256/**
232 * wait_event_timeout - sleep until a condition gets true or a timeout elapses 257 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
@@ -248,28 +273,14 @@ do { \
248#define wait_event_timeout(wq, condition, timeout) \ 273#define wait_event_timeout(wq, condition, timeout) \
249({ \ 274({ \
250 long __ret = timeout; \ 275 long __ret = timeout; \
251 if (!(condition)) \ 276 if (!___wait_cond_timeout(condition)) \
252 __wait_event_timeout(wq, condition, __ret); \ 277 __ret = __wait_event_timeout(wq, condition, timeout); \
253 __ret; \ 278 __ret; \
254}) 279})
255 280
256#define __wait_event_interruptible(wq, condition, ret) \ 281#define __wait_event_interruptible(wq, condition) \
257do { \ 282 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
258 DEFINE_WAIT(__wait); \ 283 schedule())
259 \
260 for (;;) { \
261 prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \
262 if (condition) \
263 break; \
264 if (!signal_pending(current)) { \
265 schedule(); \
266 continue; \
267 } \
268 ret = -ERESTARTSYS; \
269 break; \
270 } \
271 finish_wait(&wq, &__wait); \
272} while (0)
273 284
274/** 285/**
275 * wait_event_interruptible - sleep until a condition gets true 286 * wait_event_interruptible - sleep until a condition gets true
@@ -290,31 +301,14 @@ do { \
290({ \ 301({ \
291 int __ret = 0; \ 302 int __ret = 0; \
292 if (!(condition)) \ 303 if (!(condition)) \
293 __wait_event_interruptible(wq, condition, __ret); \ 304 __ret = __wait_event_interruptible(wq, condition); \
294 __ret; \ 305 __ret; \
295}) 306})
296 307
297#define __wait_event_interruptible_timeout(wq, condition, ret) \ 308#define __wait_event_interruptible_timeout(wq, condition, timeout) \
298do { \ 309 ___wait_event(wq, ___wait_cond_timeout(condition), \
299 DEFINE_WAIT(__wait); \ 310 TASK_INTERRUPTIBLE, 0, timeout, \
300 \ 311 __ret = schedule_timeout(__ret))
301 for (;;) { \
302 prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \
303 if (condition) \
304 break; \
305 if (!signal_pending(current)) { \
306 ret = schedule_timeout(ret); \
307 if (!ret) \
308 break; \
309 continue; \
310 } \
311 ret = -ERESTARTSYS; \
312 break; \
313 } \
314 if (!ret && (condition)) \
315 ret = 1; \
316 finish_wait(&wq, &__wait); \
317} while (0)
318 312
319/** 313/**
320 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses 314 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
@@ -337,15 +331,15 @@ do { \
337#define wait_event_interruptible_timeout(wq, condition, timeout) \ 331#define wait_event_interruptible_timeout(wq, condition, timeout) \
338({ \ 332({ \
339 long __ret = timeout; \ 333 long __ret = timeout; \
340 if (!(condition)) \ 334 if (!___wait_cond_timeout(condition)) \
341 __wait_event_interruptible_timeout(wq, condition, __ret); \ 335 __ret = __wait_event_interruptible_timeout(wq, \
336 condition, timeout); \
342 __ret; \ 337 __ret; \
343}) 338})
344 339
345#define __wait_event_hrtimeout(wq, condition, timeout, state) \ 340#define __wait_event_hrtimeout(wq, condition, timeout, state) \
346({ \ 341({ \
347 int __ret = 0; \ 342 int __ret = 0; \
348 DEFINE_WAIT(__wait); \
349 struct hrtimer_sleeper __t; \ 343 struct hrtimer_sleeper __t; \
350 \ 344 \
351 hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, \ 345 hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, \
@@ -356,25 +350,15 @@ do { \
356 current->timer_slack_ns, \ 350 current->timer_slack_ns, \
357 HRTIMER_MODE_REL); \ 351 HRTIMER_MODE_REL); \
358 \ 352 \
359 for (;;) { \ 353 __ret = ___wait_event(wq, condition, state, 0, 0, \
360 prepare_to_wait(&wq, &__wait, state); \
361 if (condition) \
362 break; \
363 if (state == TASK_INTERRUPTIBLE && \
364 signal_pending(current)) { \
365 __ret = -ERESTARTSYS; \
366 break; \
367 } \
368 if (!__t.task) { \ 354 if (!__t.task) { \
369 __ret = -ETIME; \ 355 __ret = -ETIME; \
370 break; \ 356 break; \
371 } \ 357 } \
372 schedule(); \ 358 schedule()); \
373 } \
374 \ 359 \
375 hrtimer_cancel(&__t.timer); \ 360 hrtimer_cancel(&__t.timer); \
376 destroy_hrtimer_on_stack(&__t.timer); \ 361 destroy_hrtimer_on_stack(&__t.timer); \
377 finish_wait(&wq, &__wait); \
378 __ret; \ 362 __ret; \
379}) 363})
380 364
@@ -428,33 +412,15 @@ do { \
428 __ret; \ 412 __ret; \
429}) 413})
430 414
431#define __wait_event_interruptible_exclusive(wq, condition, ret) \ 415#define __wait_event_interruptible_exclusive(wq, condition) \
432do { \ 416 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
433 DEFINE_WAIT(__wait); \ 417 schedule())
434 \
435 for (;;) { \
436 prepare_to_wait_exclusive(&wq, &__wait, \
437 TASK_INTERRUPTIBLE); \
438 if (condition) { \
439 finish_wait(&wq, &__wait); \
440 break; \
441 } \
442 if (!signal_pending(current)) { \
443 schedule(); \
444 continue; \
445 } \
446 ret = -ERESTARTSYS; \
447 abort_exclusive_wait(&wq, &__wait, \
448 TASK_INTERRUPTIBLE, NULL); \
449 break; \
450 } \
451} while (0)
452 418
453#define wait_event_interruptible_exclusive(wq, condition) \ 419#define wait_event_interruptible_exclusive(wq, condition) \
454({ \ 420({ \
455 int __ret = 0; \ 421 int __ret = 0; \
456 if (!(condition)) \ 422 if (!(condition)) \
457 __wait_event_interruptible_exclusive(wq, condition, __ret);\ 423 __ret = __wait_event_interruptible_exclusive(wq, condition);\
458 __ret; \ 424 __ret; \
459}) 425})
460 426
@@ -606,24 +572,8 @@ do { \
606 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1)) 572 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, 1))
607 573
608 574
609 575#define __wait_event_killable(wq, condition) \
610#define __wait_event_killable(wq, condition, ret) \ 576 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
611do { \
612 DEFINE_WAIT(__wait); \
613 \
614 for (;;) { \
615 prepare_to_wait(&wq, &__wait, TASK_KILLABLE); \
616 if (condition) \
617 break; \
618 if (!fatal_signal_pending(current)) { \
619 schedule(); \
620 continue; \
621 } \
622 ret = -ERESTARTSYS; \
623 break; \
624 } \
625 finish_wait(&wq, &__wait); \
626} while (0)
627 577
628/** 578/**
629 * wait_event_killable - sleep until a condition gets true 579 * wait_event_killable - sleep until a condition gets true
@@ -644,26 +594,17 @@ do { \
644({ \ 594({ \
645 int __ret = 0; \ 595 int __ret = 0; \
646 if (!(condition)) \ 596 if (!(condition)) \
647 __wait_event_killable(wq, condition, __ret); \ 597 __ret = __wait_event_killable(wq, condition); \
648 __ret; \ 598 __ret; \
649}) 599})
650 600
651 601
652#define __wait_event_lock_irq(wq, condition, lock, cmd) \ 602#define __wait_event_lock_irq(wq, condition, lock, cmd) \
653do { \ 603 (void)___wait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
654 DEFINE_WAIT(__wait); \ 604 spin_unlock_irq(&lock); \
655 \ 605 cmd; \
656 for (;;) { \ 606 schedule(); \
657 prepare_to_wait(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ 607 spin_lock_irq(&lock))
658 if (condition) \
659 break; \
660 spin_unlock_irq(&lock); \
661 cmd; \
662 schedule(); \
663 spin_lock_irq(&lock); \
664 } \
665 finish_wait(&wq, &__wait); \
666} while (0)
667 608
668/** 609/**
669 * wait_event_lock_irq_cmd - sleep until a condition gets true. The 610 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
@@ -723,26 +664,12 @@ do { \
723} while (0) 664} while (0)
724 665
725 666
726#define __wait_event_interruptible_lock_irq(wq, condition, \ 667#define __wait_event_interruptible_lock_irq(wq, condition, lock, cmd) \
727 lock, ret, cmd) \ 668 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 0, 0, \
728do { \ 669 spin_unlock_irq(&lock); \
729 DEFINE_WAIT(__wait); \ 670 cmd; \
730 \ 671 schedule(); \
731 for (;;) { \ 672 spin_lock_irq(&lock))
732 prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \
733 if (condition) \
734 break; \
735 if (signal_pending(current)) { \
736 ret = -ERESTARTSYS; \
737 break; \
738 } \
739 spin_unlock_irq(&lock); \
740 cmd; \
741 schedule(); \
742 spin_lock_irq(&lock); \
743 } \
744 finish_wait(&wq, &__wait); \
745} while (0)
746 673
747/** 674/**
748 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true. 675 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
@@ -772,10 +699,9 @@ do { \
772#define wait_event_interruptible_lock_irq_cmd(wq, condition, lock, cmd) \ 699#define wait_event_interruptible_lock_irq_cmd(wq, condition, lock, cmd) \
773({ \ 700({ \
774 int __ret = 0; \ 701 int __ret = 0; \
775 \
776 if (!(condition)) \ 702 if (!(condition)) \
777 __wait_event_interruptible_lock_irq(wq, condition, \ 703 __ret = __wait_event_interruptible_lock_irq(wq, \
778 lock, __ret, cmd); \ 704 condition, lock, cmd); \
779 __ret; \ 705 __ret; \
780}) 706})
781 707
@@ -804,39 +730,24 @@ do { \
804#define wait_event_interruptible_lock_irq(wq, condition, lock) \ 730#define wait_event_interruptible_lock_irq(wq, condition, lock) \
805({ \ 731({ \
806 int __ret = 0; \ 732 int __ret = 0; \
807 \
808 if (!(condition)) \ 733 if (!(condition)) \
809 __wait_event_interruptible_lock_irq(wq, condition, \ 734 __ret = __wait_event_interruptible_lock_irq(wq, \
810 lock, __ret, ); \ 735 condition, lock,); \
811 __ret; \ 736 __ret; \
812}) 737})
813 738
814#define __wait_event_interruptible_lock_irq_timeout(wq, condition, \ 739#define __wait_event_interruptible_lock_irq_timeout(wq, condition, \
815 lock, ret) \ 740 lock, timeout) \
816do { \ 741 ___wait_event(wq, ___wait_cond_timeout(condition), \
817 DEFINE_WAIT(__wait); \ 742 TASK_INTERRUPTIBLE, 0, timeout, \
818 \ 743 spin_unlock_irq(&lock); \
819 for (;;) { \ 744 __ret = schedule_timeout(__ret); \
820 prepare_to_wait(&wq, &__wait, TASK_INTERRUPTIBLE); \ 745 spin_lock_irq(&lock));
821 if (condition) \
822 break; \
823 if (signal_pending(current)) { \
824 ret = -ERESTARTSYS; \
825 break; \
826 } \
827 spin_unlock_irq(&lock); \
828 ret = schedule_timeout(ret); \
829 spin_lock_irq(&lock); \
830 if (!ret) \
831 break; \
832 } \
833 finish_wait(&wq, &__wait); \
834} while (0)
835 746
836/** 747/**
837 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets true or a timeout elapses. 748 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
838 * The condition is checked under the lock. This is expected 749 * true or a timeout elapses. The condition is checked under
839 * to be called with the lock taken. 750 * the lock. This is expected to be called with the lock taken.
840 * @wq: the waitqueue to wait on 751 * @wq: the waitqueue to wait on
841 * @condition: a C expression for the event to wait for 752 * @condition: a C expression for the event to wait for
842 * @lock: a locked spinlock_t, which will be released before schedule() 753 * @lock: a locked spinlock_t, which will be released before schedule()
@@ -860,11 +771,10 @@ do { \
860#define wait_event_interruptible_lock_irq_timeout(wq, condition, lock, \ 771#define wait_event_interruptible_lock_irq_timeout(wq, condition, lock, \
861 timeout) \ 772 timeout) \
862({ \ 773({ \
863 int __ret = timeout; \ 774 long __ret = timeout; \
864 \ 775 if (!___wait_cond_timeout(condition)) \
865 if (!(condition)) \ 776 __ret = __wait_event_interruptible_lock_irq_timeout( \
866 __wait_event_interruptible_lock_irq_timeout( \ 777 wq, condition, lock, timeout); \
867 wq, condition, lock, __ret); \
868 __ret; \ 778 __ret; \
869}) 779})
870 780
@@ -875,20 +785,18 @@ do { \
875 * We plan to remove these interfaces. 785 * We plan to remove these interfaces.
876 */ 786 */
877extern void sleep_on(wait_queue_head_t *q); 787extern void sleep_on(wait_queue_head_t *q);
878extern long sleep_on_timeout(wait_queue_head_t *q, 788extern long sleep_on_timeout(wait_queue_head_t *q, signed long timeout);
879 signed long timeout);
880extern void interruptible_sleep_on(wait_queue_head_t *q); 789extern void interruptible_sleep_on(wait_queue_head_t *q);
881extern long interruptible_sleep_on_timeout(wait_queue_head_t *q, 790extern long interruptible_sleep_on_timeout(wait_queue_head_t *q, signed long timeout);
882 signed long timeout);
883 791
884/* 792/*
885 * Waitqueues which are removed from the waitqueue_head at wakeup time 793 * Waitqueues which are removed from the waitqueue_head at wakeup time
886 */ 794 */
887void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state); 795void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state);
888void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state); 796void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state);
797long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state);
889void finish_wait(wait_queue_head_t *q, wait_queue_t *wait); 798void finish_wait(wait_queue_head_t *q, wait_queue_t *wait);
890void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, 799void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, unsigned int mode, void *key);
891 unsigned int mode, void *key);
892int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key); 800int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
893int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key); 801int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
894 802
@@ -934,8 +842,8 @@ int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key);
934 * One uses wait_on_bit() where one is waiting for the bit to clear, 842 * One uses wait_on_bit() where one is waiting for the bit to clear,
935 * but has no intention of setting it. 843 * but has no intention of setting it.
936 */ 844 */
937static inline int wait_on_bit(void *word, int bit, 845static inline int
938 int (*action)(void *), unsigned mode) 846wait_on_bit(void *word, int bit, int (*action)(void *), unsigned mode)
939{ 847{
940 if (!test_bit(bit, word)) 848 if (!test_bit(bit, word))
941 return 0; 849 return 0;
@@ -958,8 +866,8 @@ static inline int wait_on_bit(void *word, int bit,
958 * One uses wait_on_bit_lock() where one is waiting for the bit to 866 * One uses wait_on_bit_lock() where one is waiting for the bit to
959 * clear with the intention of setting it, and when done, clearing it. 867 * clear with the intention of setting it, and when done, clearing it.
960 */ 868 */
961static inline int wait_on_bit_lock(void *word, int bit, 869static inline int
962 int (*action)(void *), unsigned mode) 870wait_on_bit_lock(void *word, int bit, int (*action)(void *), unsigned mode)
963{ 871{
964 if (!test_and_set_bit(bit, word)) 872 if (!test_and_set_bit(bit, word))
965 return 0; 873 return 0;
@@ -983,5 +891,5 @@ int wait_on_atomic_t(atomic_t *val, int (*action)(atomic_t *), unsigned mode)
983 return 0; 891 return 0;
984 return out_of_line_wait_on_atomic_t(val, action, mode); 892 return out_of_line_wait_on_atomic_t(val, action, mode);
985} 893}
986 894
987#endif 895#endif /* _LINUX_WAIT_H */
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-18 04:57:58 -0400