aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/ABI/testing/sysfs-kernel-livepatch8
-rw-r--r--Documentation/livepatch/livepatch.txt186
-rw-r--r--include/linux/init_task.h9
-rw-r--r--include/linux/livepatch.h42
-rw-r--r--include/linux/sched.h3
-rw-r--r--kernel/fork.c3
-rw-r--r--kernel/livepatch/Makefile2
-rw-r--r--kernel/livepatch/core.c105
-rw-r--r--kernel/livepatch/patch.c59
-rw-r--r--kernel/livepatch/patch.h1
-rw-r--r--kernel/livepatch/transition.c543
-rw-r--r--kernel/livepatch/transition.h14
-rw-r--r--kernel/sched/idle.c4
-rw-r--r--samples/livepatch/livepatch-sample.c17
14 files changed, 947 insertions, 49 deletions
diff --git a/Documentation/ABI/testing/sysfs-kernel-livepatch b/Documentation/ABI/testing/sysfs-kernel-livepatch
index da87f43aec58..d5d39748382f 100644
--- a/Documentation/ABI/testing/sysfs-kernel-livepatch
+++ b/Documentation/ABI/testing/sysfs-kernel-livepatch
@@ -25,6 +25,14 @@ Description:
25 code is currently applied. Writing 0 will disable the patch 25 code is currently applied. Writing 0 will disable the patch
26 while writing 1 will re-enable the patch. 26 while writing 1 will re-enable the patch.
27 27
28What: /sys/kernel/livepatch/<patch>/transition
29Date: Feb 2017
30KernelVersion: 4.12.0
31Contact: live-patching@vger.kernel.org
32Description:
33 An attribute which indicates whether the patch is currently in
34 transition.
35
28What: /sys/kernel/livepatch/<patch>/<object> 36What: /sys/kernel/livepatch/<patch>/<object>
29Date: Nov 2014 37Date: Nov 2014
30KernelVersion: 3.19.0 38KernelVersion: 3.19.0
diff --git a/Documentation/livepatch/livepatch.txt b/Documentation/livepatch/livepatch.txt
index 9d2096c7160d..4f2aec8d4c12 100644
--- a/Documentation/livepatch/livepatch.txt
+++ b/Documentation/livepatch/livepatch.txt
@@ -72,7 +72,8 @@ example, they add a NULL pointer or a boundary check, fix a race by adding
72a missing memory barrier, or add some locking around a critical section. 72a missing memory barrier, or add some locking around a critical section.
73Most of these changes are self contained and the function presents itself 73Most of these changes are self contained and the function presents itself
74the same way to the rest of the system. In this case, the functions might 74the same way to the rest of the system. In this case, the functions might
75be updated independently one by one. 75be updated independently one by one. (This can be done by setting the
76'immediate' flag in the klp_patch struct.)
76 77
77But there are more complex fixes. For example, a patch might change 78But there are more complex fixes. For example, a patch might change
78ordering of locking in multiple functions at the same time. Or a patch 79ordering of locking in multiple functions at the same time. Or a patch
@@ -86,20 +87,141 @@ or no data are stored in the modified structures at the moment.
86The theory about how to apply functions a safe way is rather complex. 87The theory about how to apply functions a safe way is rather complex.
87The aim is to define a so-called consistency model. It attempts to define 88The aim is to define a so-called consistency model. It attempts to define
88conditions when the new implementation could be used so that the system 89conditions when the new implementation could be used so that the system
89stays consistent. The theory is not yet finished. See the discussion at 90stays consistent.
90https://lkml.kernel.org/r/20141107140458.GA21774@suse.cz 91
91 92Livepatch has a consistency model which is a hybrid of kGraft and
92The current consistency model is very simple. It guarantees that either 93kpatch: it uses kGraft's per-task consistency and syscall barrier
93the old or the new function is called. But various functions get redirected 94switching combined with kpatch's stack trace switching. There are also
94one by one without any synchronization. 95a number of fallback options which make it quite flexible.
95 96
96In other words, the current implementation _never_ modifies the behavior 97Patches are applied on a per-task basis, when the task is deemed safe to
97in the middle of the call. It is because it does _not_ rewrite the entire 98switch over. When a patch is enabled, livepatch enters into a
98function in the memory. Instead, the function gets redirected at the 99transition state where tasks are converging to the patched state.
99very beginning. But this redirection is used immediately even when 100Usually this transition state can complete in a few seconds. The same
100some other functions from the same patch have not been redirected yet. 101sequence occurs when a patch is disabled, except the tasks converge from
101 102the patched state to the unpatched state.
102See also the section "Limitations" below. 103
104An interrupt handler inherits the patched state of the task it
105interrupts. The same is true for forked tasks: the child inherits the
106patched state of the parent.
107
108Livepatch uses several complementary approaches to determine when it's
109safe to patch tasks:
110
1111. The first and most effective approach is stack checking of sleeping
112 tasks. If no affected functions are on the stack of a given task,
113 the task is patched. In most cases this will patch most or all of
114 the tasks on the first try. Otherwise it'll keep trying
115 periodically. This option is only available if the architecture has
116 reliable stacks (HAVE_RELIABLE_STACKTRACE).
117
1182. The second approach, if needed, is kernel exit switching. A
119 task is switched when it returns to user space from a system call, a
120 user space IRQ, or a signal. It's useful in the following cases:
121
122 a) Patching I/O-bound user tasks which are sleeping on an affected
123 function. In this case you have to send SIGSTOP and SIGCONT to
124 force it to exit the kernel and be patched.
125 b) Patching CPU-bound user tasks. If the task is highly CPU-bound
126 then it will get patched the next time it gets interrupted by an
127 IRQ.
128 c) In the future it could be useful for applying patches for
129 architectures which don't yet have HAVE_RELIABLE_STACKTRACE. In
130 this case you would have to signal most of the tasks on the
131 system. However this isn't supported yet because there's
132 currently no way to patch kthreads without
133 HAVE_RELIABLE_STACKTRACE.
134
1353. For idle "swapper" tasks, since they don't ever exit the kernel, they
136 instead have a klp_update_patch_state() call in the idle loop which
137 allows them to be patched before the CPU enters the idle state.
138
139 (Note there's not yet such an approach for kthreads.)
140
141All the above approaches may be skipped by setting the 'immediate' flag
142in the 'klp_patch' struct, which will disable per-task consistency and
143patch all tasks immediately. This can be useful if the patch doesn't
144change any function or data semantics. Note that, even with this flag
145set, it's possible that some tasks may still be running with an old
146version of the function, until that function returns.
147
148There's also an 'immediate' flag in the 'klp_func' struct which allows
149you to specify that certain functions in the patch can be applied
150without per-task consistency. This might be useful if you want to patch
151a common function like schedule(), and the function change doesn't need
152consistency but the rest of the patch does.
153
154For architectures which don't have HAVE_RELIABLE_STACKTRACE, the user
155must set patch->immediate which causes all tasks to be patched
156immediately. This option should be used with care, only when the patch
157doesn't change any function or data semantics.
158
159In the future, architectures which don't have HAVE_RELIABLE_STACKTRACE
160may be allowed to use per-task consistency if we can come up with
161another way to patch kthreads.
162
163The /sys/kernel/livepatch/<patch>/transition file shows whether a patch
164is in transition. Only a single patch (the topmost patch on the stack)
165can be in transition at a given time. A patch can remain in transition
166indefinitely, if any of the tasks are stuck in the initial patch state.
167
168A transition can be reversed and effectively canceled by writing the
169opposite value to the /sys/kernel/livepatch/<patch>/enabled file while
170the transition is in progress. Then all the tasks will attempt to
171converge back to the original patch state.
172
173There's also a /proc/<pid>/patch_state file which can be used to
174determine which tasks are blocking completion of a patching operation.
175If a patch is in transition, this file shows 0 to indicate the task is
176unpatched and 1 to indicate it's patched. Otherwise, if no patch is in
177transition, it shows -1. Any tasks which are blocking the transition
178can be signaled with SIGSTOP and SIGCONT to force them to change their
179patched state.
180
181
1823.1 Adding consistency model support to new architectures
183---------------------------------------------------------
184
185For adding consistency model support to new architectures, there are a
186few options:
187
1881) Add CONFIG_HAVE_RELIABLE_STACKTRACE. This means porting objtool, and
189 for non-DWARF unwinders, also making sure there's a way for the stack
190 tracing code to detect interrupts on the stack.
191
1922) Alternatively, ensure that every kthread has a call to
193 klp_update_patch_state() in a safe location. Kthreads are typically
194 in an infinite loop which does some action repeatedly. The safe
195 location to switch the kthread's patch state would be at a designated
196 point in the loop where there are no locks taken and all data
197 structures are in a well-defined state.
198
199 The location is clear when using workqueues or the kthread worker
200 API. These kthreads process independent actions in a generic loop.
201
202 It's much more complicated with kthreads which have a custom loop.
203 There the safe location must be carefully selected on a case-by-case
204 basis.
205
206 In that case, arches without HAVE_RELIABLE_STACKTRACE would still be
207 able to use the non-stack-checking parts of the consistency model:
208
209 a) patching user tasks when they cross the kernel/user space
210 boundary; and
211
212 b) patching kthreads and idle tasks at their designated patch points.
213
214 This option isn't as good as option 1 because it requires signaling
215 user tasks and waking kthreads to patch them. But it could still be
216 a good backup option for those architectures which don't have
217 reliable stack traces yet.
218
219In the meantime, patches for such architectures can bypass the
220consistency model by setting klp_patch.immediate to true. This option
221is perfectly fine for patches which don't change the semantics of the
222patched functions. In practice, this is usable for ~90% of security
223fixes. Use of this option also means the patch can't be unloaded after
224it has been disabled.
103 225
104 226
1054. Livepatch module 2274. Livepatch module
@@ -134,7 +256,7 @@ Documentation/livepatch/module-elf-format.txt for more details.
134 256
135 257
1364.2. Metadata 2584.2. Metadata
137------------ 259-------------
138 260
139The patch is described by several structures that split the information 261The patch is described by several structures that split the information
140into three levels: 262into three levels:
@@ -156,6 +278,9 @@ into three levels:
156 only for a particular object ( vmlinux or a kernel module ). Note that 278 only for a particular object ( vmlinux or a kernel module ). Note that
157 kallsyms allows for searching symbols according to the object name. 279 kallsyms allows for searching symbols according to the object name.
158 280
281 There's also an 'immediate' flag which, when set, patches the
282 function immediately, bypassing the consistency model safety checks.
283
159 + struct klp_object defines an array of patched functions (struct 284 + struct klp_object defines an array of patched functions (struct
160 klp_func) in the same object. Where the object is either vmlinux 285 klp_func) in the same object. Where the object is either vmlinux
161 (NULL) or a module name. 286 (NULL) or a module name.
@@ -172,10 +297,13 @@ into three levels:
172 This structure handles all patched functions consistently and eventually, 297 This structure handles all patched functions consistently and eventually,
173 synchronously. The whole patch is applied only when all patched 298 synchronously. The whole patch is applied only when all patched
174 symbols are found. The only exception are symbols from objects 299 symbols are found. The only exception are symbols from objects
175 (kernel modules) that have not been loaded yet. Also if a more complex 300 (kernel modules) that have not been loaded yet.
176 consistency model is supported then a selected unit (thread, 301
177 kernel as a whole) will see the new code from the entire patch 302 Setting the 'immediate' flag applies the patch to all tasks
178 only when it is in a safe state. 303 immediately, bypassing the consistency model safety checks.
304
305 For more details on how the patch is applied on a per-task basis,
306 see the "Consistency model" section.
179 307
180 308
1814.3. Livepatch module handling 3094.3. Livepatch module handling
@@ -239,9 +367,15 @@ Registered patches might be enabled either by calling klp_enable_patch() or
239by writing '1' to /sys/kernel/livepatch/<name>/enabled. The system will 367by writing '1' to /sys/kernel/livepatch/<name>/enabled. The system will
240start using the new implementation of the patched functions at this stage. 368start using the new implementation of the patched functions at this stage.
241 369
242In particular, if an original function is patched for the first time, a 370When a patch is enabled, livepatch enters into a transition state where
243function specific struct klp_ops is created and an universal ftrace handler 371tasks are converging to the patched state. This is indicated by a value
244is registered. 372of '1' in /sys/kernel/livepatch/<name>/transition. Once all tasks have
373been patched, the 'transition' value changes to '0'. For more
374information about this process, see the "Consistency model" section.
375
376If an original function is patched for the first time, a function
377specific struct klp_ops is created and an universal ftrace handler is
378registered.
245 379
246Functions might be patched multiple times. The ftrace handler is registered 380Functions might be patched multiple times. The ftrace handler is registered
247only once for the given function. Further patches just add an entry to the 381only once for the given function. Further patches just add an entry to the
@@ -261,6 +395,12 @@ by writing '0' to /sys/kernel/livepatch/<name>/enabled. At this stage
261either the code from the previously enabled patch or even the original 395either the code from the previously enabled patch or even the original
262code gets used. 396code gets used.
263 397
398When a patch is disabled, livepatch enters into a transition state where
399tasks are converging to the unpatched state. This is indicated by a
400value of '1' in /sys/kernel/livepatch/<name>/transition. Once all tasks
401have been unpatched, the 'transition' value changes to '0'. For more
402information about this process, see the "Consistency model" section.
403
264Here all the functions (struct klp_func) associated with the to-be-disabled 404Here all the functions (struct klp_func) associated with the to-be-disabled
265patch are removed from the corresponding struct klp_ops. The ftrace handler 405patch are removed from the corresponding struct klp_ops. The ftrace handler
266is unregistered and the struct klp_ops is freed when the func_stack list 406is unregistered and the struct klp_ops is freed when the func_stack list
diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 91d9049f0039..5a791055b176 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -15,6 +15,7 @@
15#include <linux/sched/autogroup.h> 15#include <linux/sched/autogroup.h>
16#include <net/net_namespace.h> 16#include <net/net_namespace.h>
17#include <linux/sched/rt.h> 17#include <linux/sched/rt.h>
18#include <linux/livepatch.h>
18#include <linux/mm_types.h> 19#include <linux/mm_types.h>
19 20
20#include <asm/thread_info.h> 21#include <asm/thread_info.h>
@@ -202,6 +203,13 @@ extern struct cred init_cred;
202# define INIT_KASAN(tsk) 203# define INIT_KASAN(tsk)
203#endif 204#endif
204 205
206#ifdef CONFIG_LIVEPATCH
207# define INIT_LIVEPATCH(tsk) \
208 .patch_state = KLP_UNDEFINED,
209#else
210# define INIT_LIVEPATCH(tsk)
211#endif
212
205#ifdef CONFIG_THREAD_INFO_IN_TASK 213#ifdef CONFIG_THREAD_INFO_IN_TASK
206# define INIT_TASK_TI(tsk) \ 214# define INIT_TASK_TI(tsk) \
207 .thread_info = INIT_THREAD_INFO(tsk), \ 215 .thread_info = INIT_THREAD_INFO(tsk), \
@@ -288,6 +296,7 @@ extern struct cred init_cred;
288 INIT_VTIME(tsk) \ 296 INIT_VTIME(tsk) \
289 INIT_NUMA_BALANCING(tsk) \ 297 INIT_NUMA_BALANCING(tsk) \
290 INIT_KASAN(tsk) \ 298 INIT_KASAN(tsk) \
299 INIT_LIVEPATCH(tsk) \
291} 300}
292 301
293 302
diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h
index 6602b34bed2b..ed90ad1605c1 100644
--- a/include/linux/livepatch.h
+++ b/include/linux/livepatch.h
@@ -28,18 +28,40 @@
28 28
29#include <asm/livepatch.h> 29#include <asm/livepatch.h>
30 30
31/* task patch states */
32#define KLP_UNDEFINED -1
33#define KLP_UNPATCHED 0
34#define KLP_PATCHED 1
35
31/** 36/**
32 * struct klp_func - function structure for live patching 37 * struct klp_func - function structure for live patching
33 * @old_name: name of the function to be patched 38 * @old_name: name of the function to be patched
34 * @new_func: pointer to the patched function code 39 * @new_func: pointer to the patched function code
35 * @old_sympos: a hint indicating which symbol position the old function 40 * @old_sympos: a hint indicating which symbol position the old function
36 * can be found (optional) 41 * can be found (optional)
42 * @immediate: patch the func immediately, bypassing safety mechanisms
37 * @old_addr: the address of the function being patched 43 * @old_addr: the address of the function being patched
38 * @kobj: kobject for sysfs resources 44 * @kobj: kobject for sysfs resources
39 * @stack_node: list node for klp_ops func_stack list 45 * @stack_node: list node for klp_ops func_stack list
40 * @old_size: size of the old function 46 * @old_size: size of the old function
41 * @new_size: size of the new function 47 * @new_size: size of the new function
42 * @patched: the func has been added to the klp_ops list 48 * @patched: the func has been added to the klp_ops list
49 * @transition: the func is currently being applied or reverted
50 *
51 * The patched and transition variables define the func's patching state. When
52 * patching, a func is always in one of the following states:
53 *
54 * patched=0 transition=0: unpatched
55 * patched=0 transition=1: unpatched, temporary starting state
56 * patched=1 transition=1: patched, may be visible to some tasks
57 * patched=1 transition=0: patched, visible to all tasks
58 *
59 * And when unpatching, it goes in the reverse order:
60 *
61 * patched=1 transition=0: patched, visible to all tasks
62 * patched=1 transition=1: patched, may be visible to some tasks
63 * patched=0 transition=1: unpatched, temporary ending state
64 * patched=0 transition=0: unpatched
43 */ 65 */
44struct klp_func { 66struct klp_func {
45 /* external */ 67 /* external */
@@ -53,6 +75,7 @@ struct klp_func {
53 * in kallsyms for the given object is used. 75 * in kallsyms for the given object is used.
54 */ 76 */
55 unsigned long old_sympos; 77 unsigned long old_sympos;
78 bool immediate;
56 79
57 /* internal */ 80 /* internal */
58 unsigned long old_addr; 81 unsigned long old_addr;
@@ -60,6 +83,7 @@ struct klp_func {
60 struct list_head stack_node; 83 struct list_head stack_node;
61 unsigned long old_size, new_size; 84 unsigned long old_size, new_size;
62 bool patched; 85 bool patched;
86 bool transition;
63}; 87};
64 88
65/** 89/**
@@ -68,7 +92,7 @@ struct klp_func {
68 * @funcs: function entries for functions to be patched in the object 92 * @funcs: function entries for functions to be patched in the object
69 * @kobj: kobject for sysfs resources 93 * @kobj: kobject for sysfs resources
70 * @mod: kernel module associated with the patched object 94 * @mod: kernel module associated with the patched object
71 * (NULL for vmlinux) 95 * (NULL for vmlinux)
72 * @patched: the object's funcs have been added to the klp_ops list 96 * @patched: the object's funcs have been added to the klp_ops list
73 */ 97 */
74struct klp_object { 98struct klp_object {
@@ -86,6 +110,7 @@ struct klp_object {
86 * struct klp_patch - patch structure for live patching 110 * struct klp_patch - patch structure for live patching
87 * @mod: reference to the live patch module 111 * @mod: reference to the live patch module
88 * @objs: object entries for kernel objects to be patched 112 * @objs: object entries for kernel objects to be patched
113 * @immediate: patch all funcs immediately, bypassing safety mechanisms
89 * @list: list node for global list of registered patches 114 * @list: list node for global list of registered patches
90 * @kobj: kobject for sysfs resources 115 * @kobj: kobject for sysfs resources
91 * @enabled: the patch is enabled (but operation may be incomplete) 116 * @enabled: the patch is enabled (but operation may be incomplete)
@@ -94,6 +119,7 @@ struct klp_patch {
94 /* external */ 119 /* external */
95 struct module *mod; 120 struct module *mod;
96 struct klp_object *objs; 121 struct klp_object *objs;
122 bool immediate;
97 123
98 /* internal */ 124 /* internal */
99 struct list_head list; 125 struct list_head list;
@@ -121,13 +147,27 @@ void arch_klp_init_object_loaded(struct klp_patch *patch,
121int klp_module_coming(struct module *mod); 147int klp_module_coming(struct module *mod);
122void klp_module_going(struct module *mod); 148void klp_module_going(struct module *mod);
123 149
150void klp_copy_process(struct task_struct *child);
124void klp_update_patch_state(struct task_struct *task); 151void klp_update_patch_state(struct task_struct *task);
125 152
153static inline bool klp_patch_pending(struct task_struct *task)
154{
155 return test_tsk_thread_flag(task, TIF_PATCH_PENDING);
156}
157
158static inline bool klp_have_reliable_stack(void)
159{
160 return IS_ENABLED(CONFIG_STACKTRACE) &&
161 IS_ENABLED(CONFIG_HAVE_RELIABLE_STACKTRACE);
162}
163
126#else /* !CONFIG_LIVEPATCH */ 164#else /* !CONFIG_LIVEPATCH */
127 165
128static inline int klp_module_coming(struct module *mod) { return 0; } 166static inline int klp_module_coming(struct module *mod) { return 0; }
129static inline void klp_module_going(struct module *mod) {} 167static inline void klp_module_going(struct module *mod) {}
168static inline bool klp_patch_pending(struct task_struct *task) { return false; }
130static inline void klp_update_patch_state(struct task_struct *task) {} 169static inline void klp_update_patch_state(struct task_struct *task) {}
170static inline void klp_copy_process(struct task_struct *child) {}
131 171
132#endif /* CONFIG_LIVEPATCH */ 172#endif /* CONFIG_LIVEPATCH */
133 173
diff --git a/include/linux/sched.h b/include/linux/sched.h
index d67eee84fd43..e11032010318 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1038,6 +1038,9 @@ struct task_struct {
1038 /* A live task holds one reference: */ 1038 /* A live task holds one reference: */
1039 atomic_t stack_refcount; 1039 atomic_t stack_refcount;
1040#endif 1040#endif
1041#ifdef CONFIG_LIVEPATCH
1042 int patch_state;
1043#endif
1041 /* CPU-specific state of this task: */ 1044 /* CPU-specific state of this task: */
1042 struct thread_struct thread; 1045 struct thread_struct thread;
1043 1046
diff --git a/kernel/fork.c b/kernel/fork.c
index 6c463c80e93d..942cbcd07c18 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -87,6 +87,7 @@
87#include <linux/compiler.h> 87#include <linux/compiler.h>
88#include <linux/sysctl.h> 88#include <linux/sysctl.h>
89#include <linux/kcov.h> 89#include <linux/kcov.h>
90#include <linux/livepatch.h>
90 91
91#include <asm/pgtable.h> 92#include <asm/pgtable.h>
92#include <asm/pgalloc.h> 93#include <asm/pgalloc.h>
@@ -1797,6 +1798,8 @@ static __latent_entropy struct task_struct *copy_process(
1797 p->parent_exec_id = current->self_exec_id; 1798 p->parent_exec_id = current->self_exec_id;
1798 } 1799 }
1799 1800
1801 klp_copy_process(p);
1802
1800 spin_lock(&current->sighand->siglock); 1803 spin_lock(&current->sighand->siglock);
1801 1804
1802 /* 1805 /*
diff --git a/kernel/livepatch/Makefile b/kernel/livepatch/Makefile
index e136dad8ff7e..2b8bdb1925da 100644
--- a/kernel/livepatch/Makefile
+++ b/kernel/livepatch/Makefile
@@ -1,3 +1,3 @@
1obj-$(CONFIG_LIVEPATCH) += livepatch.o 1obj-$(CONFIG_LIVEPATCH) += livepatch.o
2 2
3livepatch-objs := core.o patch.o 3livepatch-objs := core.o patch.o transition.o
diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c
index 10ba3a1578bd..3dc3c9049690 100644
--- a/kernel/livepatch/core.c
+++ b/kernel/livepatch/core.c
@@ -31,22 +31,22 @@
31#include <linux/moduleloader.h> 31#include <linux/moduleloader.h>
32#include <asm/cacheflush.h> 32#include <asm/cacheflush.h>
33#include "patch.h" 33#include "patch.h"
34#include "transition.h"
34 35
35/* 36/*
36 * The klp_mutex protects the global lists and state transitions of any 37 * klp_mutex is a coarse lock which serializes access to klp data. All
37 * structure reachable from them. References to any structure must be obtained 38 * accesses to klp-related variables and structures must have mutex protection,
38 * under mutex protection (except in klp_ftrace_handler(), which uses RCU to 39 * except within the following functions which carefully avoid the need for it:
39 * ensure it gets consistent data). 40 *
41 * - klp_ftrace_handler()
42 * - klp_update_patch_state()
40 */ 43 */
41static DEFINE_MUTEX(klp_mutex); 44DEFINE_MUTEX(klp_mutex);
42 45
43static LIST_HEAD(klp_patches); 46static LIST_HEAD(klp_patches);
44 47
45static struct kobject *klp_root_kobj; 48static struct kobject *klp_root_kobj;
46 49
47/* TODO: temporary stub */
48void klp_update_patch_state(struct task_struct *task) {}
49
50static bool klp_is_module(struct klp_object *obj) 50static bool klp_is_module(struct klp_object *obj)
51{ 51{
52 return obj->name; 52 return obj->name;
@@ -85,7 +85,6 @@ static void klp_find_object_module(struct klp_object *obj)
85 mutex_unlock(&module_mutex); 85 mutex_unlock(&module_mutex);
86} 86}
87 87
88/* klp_mutex must be held by caller */
89static bool klp_is_patch_registered(struct klp_patch *patch) 88static bool klp_is_patch_registered(struct klp_patch *patch)
90{ 89{
91 struct klp_patch *mypatch; 90 struct klp_patch *mypatch;
@@ -281,20 +280,27 @@ static int klp_write_object_relocations(struct module *pmod,
281 280
282static int __klp_disable_patch(struct klp_patch *patch) 281static int __klp_disable_patch(struct klp_patch *patch)
283{ 282{
284 struct klp_object *obj; 283 if (klp_transition_patch)
284 return -EBUSY;
285 285
286 /* enforce stacking: only the last enabled patch can be disabled */ 286 /* enforce stacking: only the last enabled patch can be disabled */
287 if (!list_is_last(&patch->list, &klp_patches) && 287 if (!list_is_last(&patch->list, &klp_patches) &&
288 list_next_entry(patch, list)->enabled) 288 list_next_entry(patch, list)->enabled)
289 return -EBUSY; 289 return -EBUSY;
290 290
291 pr_notice("disabling patch '%s'\n", patch->mod->name); 291 klp_init_transition(patch, KLP_UNPATCHED);
292 292
293 klp_for_each_object(patch, obj) { 293 /*
294 if (obj->patched) 294 * Enforce the order of the func->transition writes in
295 klp_unpatch_object(obj); 295 * klp_init_transition() and the TIF_PATCH_PENDING writes in
296 } 296 * klp_start_transition(). In the rare case where klp_ftrace_handler()
297 * is called shortly after klp_update_patch_state() switches the task,
298 * this ensures the handler sees that func->transition is set.
299 */
300 smp_wmb();
297 301
302 klp_start_transition();
303 klp_try_complete_transition();
298 patch->enabled = false; 304 patch->enabled = false;
299 305
300 return 0; 306 return 0;
@@ -337,6 +343,9 @@ static int __klp_enable_patch(struct klp_patch *patch)
337 struct klp_object *obj; 343 struct klp_object *obj;
338 int ret; 344 int ret;
339 345
346 if (klp_transition_patch)
347 return -EBUSY;
348
340 if (WARN_ON(patch->enabled)) 349 if (WARN_ON(patch->enabled))
341 return -EINVAL; 350 return -EINVAL;
342 351
@@ -347,22 +356,36 @@ static int __klp_enable_patch(struct klp_patch *patch)
347 356
348 pr_notice("enabling patch '%s'\n", patch->mod->name); 357 pr_notice("enabling patch '%s'\n", patch->mod->name);
349 358
359 klp_init_transition(patch, KLP_PATCHED);
360
361 /*
362 * Enforce the order of the func->transition writes in
363 * klp_init_transition() and the ops->func_stack writes in
364 * klp_patch_object(), so that klp_ftrace_handler() will see the
365 * func->transition updates before the handler is registered and the
366 * new funcs become visible to the handler.
367 */
368 smp_wmb();
369
350 klp_for_each_object(patch, obj) { 370 klp_for_each_object(patch, obj) {
351 if (!klp_is_object_loaded(obj)) 371 if (!klp_is_object_loaded(obj))
352 continue; 372 continue;
353 373
354 ret = klp_patch_object(obj); 374 ret = klp_patch_object(obj);
355 if (ret) 375 if (ret) {
356 goto unregister; 376 pr_warn("failed to enable patch '%s'\n",
377 patch->mod->name);
378
379 klp_cancel_transition();
380 return ret;
381 }
357 } 382 }
358 383
384 klp_start_transition();
385 klp_try_complete_transition();
359 patch->enabled = true; 386 patch->enabled = true;
360 387
361 return 0; 388 return 0;
362
363unregister:
364 WARN_ON(__klp_disable_patch(patch));
365 return ret;
366} 389}
367 390
368/** 391/**
@@ -399,6 +422,7 @@ EXPORT_SYMBOL_GPL(klp_enable_patch);
399 * /sys/kernel/livepatch 422 * /sys/kernel/livepatch
400 * /sys/kernel/livepatch/<patch> 423 * /sys/kernel/livepatch/<patch>
401 * /sys/kernel/livepatch/<patch>/enabled 424 * /sys/kernel/livepatch/<patch>/enabled
425 * /sys/kernel/livepatch/<patch>/transition
402 * /sys/kernel/livepatch/<patch>/<object> 426 * /sys/kernel/livepatch/<patch>/<object>
403 * /sys/kernel/livepatch/<patch>/<object>/<function,sympos> 427 * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
404 */ 428 */
@@ -424,7 +448,9 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
424 goto err; 448 goto err;
425 } 449 }
426 450
427 if (enabled) { 451 if (patch == klp_transition_patch) {
452 klp_reverse_transition();
453 } else if (enabled) {
428 ret = __klp_enable_patch(patch); 454 ret = __klp_enable_patch(patch);
429 if (ret) 455 if (ret)
430 goto err; 456 goto err;
@@ -452,9 +478,21 @@ static ssize_t enabled_show(struct kobject *kobj,
452 return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled); 478 return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
453} 479}
454 480
481static ssize_t transition_show(struct kobject *kobj,
482 struct kobj_attribute *attr, char *buf)
483{
484 struct klp_patch *patch;
485
486 patch = container_of(kobj, struct klp_patch, kobj);
487 return snprintf(buf, PAGE_SIZE-1, "%d\n",
488 patch == klp_transition_patch);
489}
490
455static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled); 491static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
492static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
456static struct attribute *klp_patch_attrs[] = { 493static struct attribute *klp_patch_attrs[] = {
457 &enabled_kobj_attr.attr, 494 &enabled_kobj_attr.attr,
495 &transition_kobj_attr.attr,
458 NULL 496 NULL
459}; 497};
460 498
@@ -544,6 +582,7 @@ static int klp_init_func(struct klp_object *obj, struct klp_func *func)
544 582
545 INIT_LIST_HEAD(&func->stack_node); 583 INIT_LIST_HEAD(&func->stack_node);
546 func->patched = false; 584 func->patched = false;
585 func->transition = false;
547 586
548 /* The format for the sysfs directory is <function,sympos> where sympos 587 /* The format for the sysfs directory is <function,sympos> where sympos
549 * is the nth occurrence of this symbol in kallsyms for the patched 588 * is the nth occurrence of this symbol in kallsyms for the patched
@@ -740,6 +779,16 @@ int klp_register_patch(struct klp_patch *patch)
740 return -ENODEV; 779 return -ENODEV;
741 780
742 /* 781 /*
782 * Architectures without reliable stack traces have to set
783 * patch->immediate because there's currently no way to patch kthreads
784 * with the consistency model.
785 */
786 if (!klp_have_reliable_stack() && !patch->immediate) {
787 pr_err("This architecture doesn't have support for the livepatch consistency model.\n");
788 return -ENOSYS;
789 }
790
791 /*
743 * A reference is taken on the patch module to prevent it from being 792 * A reference is taken on the patch module to prevent it from being
744 * unloaded. Right now, we don't allow patch modules to unload since 793 * unloaded. Right now, we don't allow patch modules to unload since
745 * there is currently no method to determine if a thread is still 794 * there is currently no method to determine if a thread is still
@@ -788,7 +837,11 @@ int klp_module_coming(struct module *mod)
788 goto err; 837 goto err;
789 } 838 }
790 839
791 if (!patch->enabled) 840 /*
841 * Only patch the module if the patch is enabled or is
842 * in transition.
843 */
844 if (!patch->enabled && patch != klp_transition_patch)
792 break; 845 break;
793 846
794 pr_notice("applying patch '%s' to loading module '%s'\n", 847 pr_notice("applying patch '%s' to loading module '%s'\n",
@@ -845,7 +898,11 @@ void klp_module_going(struct module *mod)
845 if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) 898 if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
846 continue; 899 continue;
847 900
848 if (patch->enabled) { 901 /*
902 * Only unpatch the module if the patch is enabled or
903 * is in transition.
904 */
905 if (patch->enabled || patch == klp_transition_patch) {
849 pr_notice("reverting patch '%s' on unloading module '%s'\n", 906 pr_notice("reverting patch '%s' on unloading module '%s'\n",
850 patch->mod->name, obj->mod->name); 907 patch->mod->name, obj->mod->name);
851 klp_unpatch_object(obj); 908 klp_unpatch_object(obj);
diff --git a/kernel/livepatch/patch.c b/kernel/livepatch/patch.c
index 5efa2620851a..f8269036bf0b 100644
--- a/kernel/livepatch/patch.c
+++ b/kernel/livepatch/patch.c
@@ -29,6 +29,7 @@
29#include <linux/bug.h> 29#include <linux/bug.h>
30#include <linux/printk.h> 30#include <linux/printk.h>
31#include "patch.h" 31#include "patch.h"
32#include "transition.h"
32 33
33static LIST_HEAD(klp_ops); 34static LIST_HEAD(klp_ops);
34 35
@@ -54,15 +55,64 @@ static void notrace klp_ftrace_handler(unsigned long ip,
54{ 55{
55 struct klp_ops *ops; 56 struct klp_ops *ops;
56 struct klp_func *func; 57 struct klp_func *func;
58 int patch_state;
57 59
58 ops = container_of(fops, struct klp_ops, fops); 60 ops = container_of(fops, struct klp_ops, fops);
59 61
60 rcu_read_lock(); 62 rcu_read_lock();
63
61 func = list_first_or_null_rcu(&ops->func_stack, struct klp_func, 64 func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
62 stack_node); 65 stack_node);
66
67 /*
68 * func should never be NULL because preemption should be disabled here
69 * and unregister_ftrace_function() does the equivalent of a
70 * synchronize_sched() before the func_stack removal.
71 */
63 if (WARN_ON_ONCE(!func)) 72 if (WARN_ON_ONCE(!func))
64 goto unlock; 73 goto unlock;
65 74
75 /*
76 * In the enable path, enforce the order of the ops->func_stack and
77 * func->transition reads. The corresponding write barrier is in
78 * __klp_enable_patch().
79 *
80 * (Note that this barrier technically isn't needed in the disable
81 * path. In the rare case where klp_update_patch_state() runs before
82 * this handler, its TIF_PATCH_PENDING read and this func->transition
83 * read need to be ordered. But klp_update_patch_state() already
84 * enforces that.)
85 */
86 smp_rmb();
87
88 if (unlikely(func->transition)) {
89
90 /*
91 * Enforce the order of the func->transition and
92 * current->patch_state reads. Otherwise we could read an
93 * out-of-date task state and pick the wrong function. The
94 * corresponding write barrier is in klp_init_transition().
95 */
96 smp_rmb();
97
98 patch_state = current->patch_state;
99
100 WARN_ON_ONCE(patch_state == KLP_UNDEFINED);
101
102 if (patch_state == KLP_UNPATCHED) {
103 /*
104 * Use the previously patched version of the function.
105 * If no previous patches exist, continue with the
106 * original function.
107 */
108 func = list_entry_rcu(func->stack_node.next,
109 struct klp_func, stack_node);
110
111 if (&func->stack_node == &ops->func_stack)
112 goto unlock;
113 }
114 }
115
66 klp_arch_set_pc(regs, (unsigned long)func->new_func); 116 klp_arch_set_pc(regs, (unsigned long)func->new_func);
67unlock: 117unlock:
68 rcu_read_unlock(); 118 rcu_read_unlock();
@@ -211,3 +261,12 @@ int klp_patch_object(struct klp_object *obj)
211 261
212 return 0; 262 return 0;
213} 263}
264
265void klp_unpatch_objects(struct klp_patch *patch)
266{
267 struct klp_object *obj;
268
269 klp_for_each_object(patch, obj)
270 if (obj->patched)
271 klp_unpatch_object(obj);
272}
diff --git a/kernel/livepatch/patch.h b/kernel/livepatch/patch.h
index 2d0cce02dade..0db227170c36 100644
--- a/kernel/livepatch/patch.h
+++ b/kernel/livepatch/patch.h
@@ -28,5 +28,6 @@ struct klp_ops *klp_find_ops(unsigned long old_addr);
28 28
29int klp_patch_object(struct klp_object *obj); 29int klp_patch_object(struct klp_object *obj);
30void klp_unpatch_object(struct klp_object *obj); 30void klp_unpatch_object(struct klp_object *obj);
31void klp_unpatch_objects(struct klp_patch *patch);
31 32
32#endif /* _LIVEPATCH_PATCH_H */ 33#endif /* _LIVEPATCH_PATCH_H */
diff --git a/kernel/livepatch/transition.c b/kernel/livepatch/transition.c
new file mode 100644
index 000000000000..428533ec51b5
--- /dev/null
+++ b/kernel/livepatch/transition.c
@@ -0,0 +1,543 @@
1/*
2 * transition.c - Kernel Live Patching transition functions
3 *
4 * Copyright (C) 2015-2016 Josh Poimboeuf <jpoimboe@redhat.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21
22#include <linux/cpu.h>
23#include <linux/stacktrace.h>
24#include "patch.h"
25#include "transition.h"
26#include "../sched/sched.h"
27
28#define MAX_STACK_ENTRIES 100
29#define STACK_ERR_BUF_SIZE 128
30
31extern struct mutex klp_mutex;
32
33struct klp_patch *klp_transition_patch;
34
35static int klp_target_state = KLP_UNDEFINED;
36
37/*
38 * This work can be performed periodically to finish patching or unpatching any
39 * "straggler" tasks which failed to transition in the first attempt.
40 */
41static void klp_transition_work_fn(struct work_struct *work)
42{
43 mutex_lock(&klp_mutex);
44
45 if (klp_transition_patch)
46 klp_try_complete_transition();
47
48 mutex_unlock(&klp_mutex);
49}
50static DECLARE_DELAYED_WORK(klp_transition_work, klp_transition_work_fn);
51
52/*
53 * The transition to the target patch state is complete. Clean up the data
54 * structures.
55 */
56static void klp_complete_transition(void)
57{
58 struct klp_object *obj;
59 struct klp_func *func;
60 struct task_struct *g, *task;
61 unsigned int cpu;
62
63 if (klp_target_state == KLP_UNPATCHED) {
64 /*
65 * All tasks have transitioned to KLP_UNPATCHED so we can now
66 * remove the new functions from the func_stack.
67 */
68 klp_unpatch_objects(klp_transition_patch);
69
70 /*
71 * Make sure klp_ftrace_handler() can no longer see functions
72 * from this patch on the ops->func_stack. Otherwise, after
73 * func->transition gets cleared, the handler may choose a
74 * removed function.
75 */
76 synchronize_rcu();
77 }
78
79 if (klp_transition_patch->immediate)
80 goto done;
81
82 klp_for_each_object(klp_transition_patch, obj)
83 klp_for_each_func(obj, func)
84 func->transition = false;
85
86 /* Prevent klp_ftrace_handler() from seeing KLP_UNDEFINED state */
87 if (klp_target_state == KLP_PATCHED)
88 synchronize_rcu();
89
90 read_lock(&tasklist_lock);
91 for_each_process_thread(g, task) {
92 WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
93 task->patch_state = KLP_UNDEFINED;
94 }
95 read_unlock(&tasklist_lock);
96
97 for_each_possible_cpu(cpu) {
98 task = idle_task(cpu);
99 WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING));
100 task->patch_state = KLP_UNDEFINED;
101 }
102
103done:
104 klp_target_state = KLP_UNDEFINED;
105 klp_transition_patch = NULL;
106}
107
108/*
109 * This is called in the error path, to cancel a transition before it has
110 * started, i.e. klp_init_transition() has been called but
111 * klp_start_transition() hasn't. If the transition *has* been started,
112 * klp_reverse_transition() should be used instead.
113 */
114void klp_cancel_transition(void)
115{
116 klp_target_state = !klp_target_state;
117 klp_complete_transition();
118}
119
120/*
121 * Switch the patched state of the task to the set of functions in the target
122 * patch state.
123 *
124 * NOTE: If task is not 'current', the caller must ensure the task is inactive.
125 * Otherwise klp_ftrace_handler() might read the wrong 'patch_state' value.
126 */
127void klp_update_patch_state(struct task_struct *task)
128{
129 rcu_read_lock();
130
131 /*
132 * This test_and_clear_tsk_thread_flag() call also serves as a read
133 * barrier (smp_rmb) for two cases:
134 *
135 * 1) Enforce the order of the TIF_PATCH_PENDING read and the
136 * klp_target_state read. The corresponding write barrier is in
137 * klp_init_transition().
138 *
139 * 2) Enforce the order of the TIF_PATCH_PENDING read and a future read
140 * of func->transition, if klp_ftrace_handler() is called later on
141 * the same CPU. See __klp_disable_patch().
142 */
143 if (test_and_clear_tsk_thread_flag(task, TIF_PATCH_PENDING))
144 task->patch_state = READ_ONCE(klp_target_state);
145
146 rcu_read_unlock();
147}
148
149/*
150 * Determine whether the given stack trace includes any references to a
151 * to-be-patched or to-be-unpatched function.
152 */
153static int klp_check_stack_func(struct klp_func *func,
154 struct stack_trace *trace)
155{
156 unsigned long func_addr, func_size, address;
157 struct klp_ops *ops;
158 int i;
159
160 if (func->immediate)
161 return 0;
162
163 for (i = 0; i < trace->nr_entries; i++) {
164 address = trace->entries[i];
165
166 if (klp_target_state == KLP_UNPATCHED) {
167 /*
168 * Check for the to-be-unpatched function
169 * (the func itself).
170 */
171 func_addr = (unsigned long)func->new_func;
172 func_size = func->new_size;
173 } else {
174 /*
175 * Check for the to-be-patched function
176 * (the previous func).
177 */
178 ops = klp_find_ops(func->old_addr);
179
180 if (list_is_singular(&ops->func_stack)) {
181 /* original function */
182 func_addr = func->old_addr;
183 func_size = func->old_size;
184 } else {
185 /* previously patched function */
186 struct klp_func *prev;
187
188 prev = list_next_entry(func, stack_node);
189 func_addr = (unsigned long)prev->new_func;
190 func_size = prev->new_size;
191 }
192 }
193
194 if (address >= func_addr && address < func_addr + func_size)
195 return -EAGAIN;
196 }
197
198 return 0;
199}
200
201/*
202 * Determine whether it's safe to transition the task to the target patch state
203 * by looking for any to-be-patched or to-be-unpatched functions on its stack.
204 */
205static int klp_check_stack(struct task_struct *task, char *err_buf)
206{
207 static unsigned long entries[MAX_STACK_ENTRIES];
208 struct stack_trace trace;
209 struct klp_object *obj;
210 struct klp_func *func;
211 int ret;
212
213 trace.skip = 0;
214 trace.nr_entries = 0;
215 trace.max_entries = MAX_STACK_ENTRIES;
216 trace.entries = entries;
217 ret = save_stack_trace_tsk_reliable(task, &trace);
218 WARN_ON_ONCE(ret == -ENOSYS);
219 if (ret) {
220 snprintf(err_buf, STACK_ERR_BUF_SIZE,
221 "%s: %s:%d has an unreliable stack\n",
222 __func__, task->comm, task->pid);
223 return ret;
224 }
225
226 klp_for_each_object(klp_transition_patch, obj) {
227 if (!obj->patched)
228 continue;
229 klp_for_each_func(obj, func) {
230 ret = klp_check_stack_func(func, &trace);
231 if (ret) {
232 snprintf(err_buf, STACK_ERR_BUF_SIZE,
233 "%s: %s:%d is sleeping on function %s\n",
234 __func__, task->comm, task->pid,
235 func->old_name);
236 return ret;
237 }
238 }
239 }
240
241 return 0;
242}
243
244/*
245 * Try to safely switch a task to the target patch state. If it's currently
246 * running, or it's sleeping on a to-be-patched or to-be-unpatched function, or
247 * if the stack is unreliable, return false.
248 */
249static bool klp_try_switch_task(struct task_struct *task)
250{
251 struct rq *rq;
252 struct rq_flags flags;
253 int ret;
254 bool success = false;
255 char err_buf[STACK_ERR_BUF_SIZE];
256
257 err_buf[0] = '\0';
258
259 /* check if this task has already switched over */
260 if (task->patch_state == klp_target_state)
261 return true;
262
263 /*
264 * For arches which don't have reliable stack traces, we have to rely
265 * on other methods (e.g., switching tasks at kernel exit).
266 */
267 if (!klp_have_reliable_stack())
268 return false;
269
270 /*
271 * Now try to check the stack for any to-be-patched or to-be-unpatched
272 * functions. If all goes well, switch the task to the target patch
273 * state.
274 */
275 rq = task_rq_lock(task, &flags);
276
277 if (task_running(rq, task) && task != current) {
278 snprintf(err_buf, STACK_ERR_BUF_SIZE,
279 "%s: %s:%d is running\n", __func__, task->comm,
280 task->pid);
281 goto done;
282 }
283
284 ret = klp_check_stack(task, err_buf);
285 if (ret)
286 goto done;
287
288 success = true;
289
290 clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
291 task->patch_state = klp_target_state;
292
293done:
294 task_rq_unlock(rq, task, &flags);
295
296 /*
297 * Due to console deadlock issues, pr_debug() can't be used while
298 * holding the task rq lock. Instead we have to use a temporary buffer
299 * and print the debug message after releasing the lock.
300 */
301 if (err_buf[0] != '\0')
302 pr_debug("%s", err_buf);
303
304 return success;
305
306}
307
308/*
309 * Try to switch all remaining tasks to the target patch state by walking the
310 * stacks of sleeping tasks and looking for any to-be-patched or
311 * to-be-unpatched functions. If such functions are found, the task can't be
312 * switched yet.
313 *
314 * If any tasks are still stuck in the initial patch state, schedule a retry.
315 */
316void klp_try_complete_transition(void)
317{
318 unsigned int cpu;
319 struct task_struct *g, *task;
320 bool complete = true;
321
322 WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
323
324 /*
325 * If the patch can be applied or reverted immediately, skip the
326 * per-task transitions.
327 */
328 if (klp_transition_patch->immediate)
329 goto success;
330
331 /*
332 * Try to switch the tasks to the target patch state by walking their
333 * stacks and looking for any to-be-patched or to-be-unpatched
334 * functions. If such functions are found on a stack, or if the stack
335 * is deemed unreliable, the task can't be switched yet.
336 *
337 * Usually this will transition most (or all) of the tasks on a system
338 * unless the patch includes changes to a very common function.
339 */
340 read_lock(&tasklist_lock);
341 for_each_process_thread(g, task)
342 if (!klp_try_switch_task(task))
343 complete = false;
344 read_unlock(&tasklist_lock);
345
346 /*
347 * Ditto for the idle "swapper" tasks.
348 */
349 get_online_cpus();
350 for_each_possible_cpu(cpu) {
351 task = idle_task(cpu);
352 if (cpu_online(cpu)) {
353 if (!klp_try_switch_task(task))
354 complete = false;
355 } else if (task->patch_state != klp_target_state) {
356 /* offline idle tasks can be switched immediately */
357 clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
358 task->patch_state = klp_target_state;
359 }
360 }
361 put_online_cpus();
362
363 if (!complete) {
364 /*
365 * Some tasks weren't able to be switched over. Try again
366 * later and/or wait for other methods like kernel exit
367 * switching.
368 */
369 schedule_delayed_work(&klp_transition_work,
370 round_jiffies_relative(HZ));
371 return;
372 }
373
374success:
375 pr_notice("'%s': %s complete\n", klp_transition_patch->mod->name,
376 klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
377
378 /* we're done, now cleanup the data structures */
379 klp_complete_transition();
380}
381
382/*
383 * Start the transition to the specified target patch state so tasks can begin
384 * switching to it.
385 */
386void klp_start_transition(void)
387{
388 struct task_struct *g, *task;
389 unsigned int cpu;
390
391 WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED);
392
393 pr_notice("'%s': %s...\n", klp_transition_patch->mod->name,
394 klp_target_state == KLP_PATCHED ? "patching" : "unpatching");
395
396 /*
397 * If the patch can be applied or reverted immediately, skip the
398 * per-task transitions.
399 */
400 if (klp_transition_patch->immediate)
401 return;
402
403 /*
404 * Mark all normal tasks as needing a patch state update. They'll
405 * switch either in klp_try_complete_transition() or as they exit the
406 * kernel.
407 */
408 read_lock(&tasklist_lock);
409 for_each_process_thread(g, task)
410 if (task->patch_state != klp_target_state)
411 set_tsk_thread_flag(task, TIF_PATCH_PENDING);
412 read_unlock(&tasklist_lock);
413
414 /*
415 * Mark all idle tasks as needing a patch state update. They'll switch
416 * either in klp_try_complete_transition() or at the idle loop switch
417 * point.
418 */
419 for_each_possible_cpu(cpu) {
420 task = idle_task(cpu);
421 if (task->patch_state != klp_target_state)
422 set_tsk_thread_flag(task, TIF_PATCH_PENDING);
423 }
424}
425
426/*
427 * Initialize the global target patch state and all tasks to the initial patch
428 * state, and initialize all function transition states to true in preparation
429 * for patching or unpatching.
430 */
431void klp_init_transition(struct klp_patch *patch, int state)
432{
433 struct task_struct *g, *task;
434 unsigned int cpu;
435 struct klp_object *obj;
436 struct klp_func *func;
437 int initial_state = !state;
438
439 WARN_ON_ONCE(klp_target_state != KLP_UNDEFINED);
440
441 klp_transition_patch = patch;
442
443 /*
444 * Set the global target patch state which tasks will switch to. This
445 * has no effect until the TIF_PATCH_PENDING flags get set later.
446 */
447 klp_target_state = state;
448
449 /*
450 * If the patch can be applied or reverted immediately, skip the
451 * per-task transitions.
452 */
453 if (patch->immediate)
454 return;
455
456 /*
457 * Initialize all tasks to the initial patch state to prepare them for
458 * switching to the target state.
459 */
460 read_lock(&tasklist_lock);
461 for_each_process_thread(g, task) {
462 WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
463 task->patch_state = initial_state;
464 }
465 read_unlock(&tasklist_lock);
466
467 /*
468 * Ditto for the idle "swapper" tasks.
469 */
470 for_each_possible_cpu(cpu) {
471 task = idle_task(cpu);
472 WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED);
473 task->patch_state = initial_state;
474 }
475
476 /*
477 * Enforce the order of the task->patch_state initializations and the
478 * func->transition updates to ensure that klp_ftrace_handler() doesn't
479 * see a func in transition with a task->patch_state of KLP_UNDEFINED.
480 *
481 * Also enforce the order of the klp_target_state write and future
482 * TIF_PATCH_PENDING writes to ensure klp_update_patch_state() doesn't
483 * set a task->patch_state to KLP_UNDEFINED.
484 */
485 smp_wmb();
486
487 /*
488 * Set the func transition states so klp_ftrace_handler() will know to
489 * switch to the transition logic.
490 *
491 * When patching, the funcs aren't yet in the func_stack and will be
492 * made visible to the ftrace handler shortly by the calls to
493 * klp_patch_object().
494 *
495 * When unpatching, the funcs are already in the func_stack and so are
496 * already visible to the ftrace handler.
497 */
498 klp_for_each_object(patch, obj)
499 klp_for_each_func(obj, func)
500 func->transition = true;
501}
502
503/*
504 * This function can be called in the middle of an existing transition to
505 * reverse the direction of the target patch state. This can be done to
506 * effectively cancel an existing enable or disable operation if there are any
507 * tasks which are stuck in the initial patch state.
508 */
509void klp_reverse_transition(void)
510{
511 unsigned int cpu;
512 struct task_struct *g, *task;
513
514 klp_transition_patch->enabled = !klp_transition_patch->enabled;
515
516 klp_target_state = !klp_target_state;
517
518 /*
519 * Clear all TIF_PATCH_PENDING flags to prevent races caused by
520 * klp_update_patch_state() running in parallel with
521 * klp_start_transition().
522 */
523 read_lock(&tasklist_lock);
524 for_each_process_thread(g, task)
525 clear_tsk_thread_flag(task, TIF_PATCH_PENDING);
526 read_unlock(&tasklist_lock);
527
528 for_each_possible_cpu(cpu)
529 clear_tsk_thread_flag(idle_task(cpu), TIF_PATCH_PENDING);
530
531 /* Let any remaining calls to klp_update_patch_state() complete */
532 synchronize_rcu();
533
534 klp_start_transition();
535}
536
537/* Called from copy_process() during fork */
538void klp_copy_process(struct task_struct *child)
539{
540 child->patch_state = current->patch_state;
541
542 /* TIF_PATCH_PENDING gets copied in setup_thread_stack() */
543}
diff --git a/kernel/livepatch/transition.h b/kernel/livepatch/transition.h
new file mode 100644
index 000000000000..ce09b326546c
--- /dev/null
+++ b/kernel/livepatch/transition.h
@@ -0,0 +1,14 @@
1#ifndef _LIVEPATCH_TRANSITION_H
2#define _LIVEPATCH_TRANSITION_H
3
4#include <linux/livepatch.h>
5
6extern struct klp_patch *klp_transition_patch;
7
8void klp_init_transition(struct klp_patch *patch, int state);
9void klp_cancel_transition(void);
10void klp_start_transition(void);
11void klp_try_complete_transition(void);
12void klp_reverse_transition(void);
13
14#endif /* _LIVEPATCH_TRANSITION_H */
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index ac6d5176463d..2a25a9ec2c6e 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -10,6 +10,7 @@
10#include <linux/mm.h> 10#include <linux/mm.h>
11#include <linux/stackprotector.h> 11#include <linux/stackprotector.h>
12#include <linux/suspend.h> 12#include <linux/suspend.h>
13#include <linux/livepatch.h>
13 14
14#include <asm/tlb.h> 15#include <asm/tlb.h>
15 16
@@ -265,6 +266,9 @@ static void do_idle(void)
265 266
266 sched_ttwu_pending(); 267 sched_ttwu_pending();
267 schedule_preempt_disabled(); 268 schedule_preempt_disabled();
269
270 if (unlikely(klp_patch_pending(current)))
271 klp_update_patch_state(current);
268} 272}
269 273
270bool cpu_in_idle(unsigned long pc) 274bool cpu_in_idle(unsigned long pc)
diff --git a/samples/livepatch/livepatch-sample.c b/samples/livepatch/livepatch-sample.c
index e34f871e69b1..629e0dca0887 100644
--- a/samples/livepatch/livepatch-sample.c
+++ b/samples/livepatch/livepatch-sample.c
@@ -17,6 +17,8 @@
17 * along with this program; if not, see <http://www.gnu.org/licenses/>. 17 * along with this program; if not, see <http://www.gnu.org/licenses/>.
18 */ 18 */
19 19
20#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21
20#include <linux/module.h> 22#include <linux/module.h>
21#include <linux/kernel.h> 23#include <linux/kernel.h>
22#include <linux/livepatch.h> 24#include <linux/livepatch.h>
@@ -69,6 +71,21 @@ static int livepatch_init(void)
69{ 71{
70 int ret; 72 int ret;
71 73
74 if (!klp_have_reliable_stack() && !patch.immediate) {
75 /*
76 * WARNING: Be very careful when using 'patch.immediate' in
77 * your patches. It's ok to use it for simple patches like
78 * this, but for more complex patches which change function
79 * semantics, locking semantics, or data structures, it may not
80 * be safe. Use of this option will also prevent removal of
81 * the patch.
82 *
83 * See Documentation/livepatch/livepatch.txt for more details.
84 */
85 patch.immediate = true;
86 pr_notice("The consistency model isn't supported for your architecture. Bypassing safety mechanisms and applying the patch immediately.\n");
87 }
88
72 ret = klp_register_patch(&patch); 89 ret = klp_register_patch(&patch);
73 if (ret) 90 if (ret)
74 return ret; 91 return ret;