diff options
-rw-r--r-- | Documentation/ABI/testing/sysfs-kernel-livepatch | 8 | ||||
-rw-r--r-- | Documentation/livepatch/livepatch.txt | 186 | ||||
-rw-r--r-- | include/linux/init_task.h | 9 | ||||
-rw-r--r-- | include/linux/livepatch.h | 42 | ||||
-rw-r--r-- | include/linux/sched.h | 3 | ||||
-rw-r--r-- | kernel/fork.c | 3 | ||||
-rw-r--r-- | kernel/livepatch/Makefile | 2 | ||||
-rw-r--r-- | kernel/livepatch/core.c | 105 | ||||
-rw-r--r-- | kernel/livepatch/patch.c | 59 | ||||
-rw-r--r-- | kernel/livepatch/patch.h | 1 | ||||
-rw-r--r-- | kernel/livepatch/transition.c | 543 | ||||
-rw-r--r-- | kernel/livepatch/transition.h | 14 | ||||
-rw-r--r-- | kernel/sched/idle.c | 4 | ||||
-rw-r--r-- | samples/livepatch/livepatch-sample.c | 17 |
14 files changed, 947 insertions, 49 deletions
diff --git a/Documentation/ABI/testing/sysfs-kernel-livepatch b/Documentation/ABI/testing/sysfs-kernel-livepatch index da87f43aec58..d5d39748382f 100644 --- a/Documentation/ABI/testing/sysfs-kernel-livepatch +++ b/Documentation/ABI/testing/sysfs-kernel-livepatch | |||
@@ -25,6 +25,14 @@ Description: | |||
25 | code is currently applied. Writing 0 will disable the patch | 25 | code is currently applied. Writing 0 will disable the patch |
26 | while writing 1 will re-enable the patch. | 26 | while writing 1 will re-enable the patch. |
27 | 27 | ||
28 | What: /sys/kernel/livepatch/<patch>/transition | ||
29 | Date: Feb 2017 | ||
30 | KernelVersion: 4.12.0 | ||
31 | Contact: live-patching@vger.kernel.org | ||
32 | Description: | ||
33 | An attribute which indicates whether the patch is currently in | ||
34 | transition. | ||
35 | |||
28 | What: /sys/kernel/livepatch/<patch>/<object> | 36 | What: /sys/kernel/livepatch/<patch>/<object> |
29 | Date: Nov 2014 | 37 | Date: Nov 2014 |
30 | KernelVersion: 3.19.0 | 38 | KernelVersion: 3.19.0 |
diff --git a/Documentation/livepatch/livepatch.txt b/Documentation/livepatch/livepatch.txt index 9d2096c7160d..4f2aec8d4c12 100644 --- a/Documentation/livepatch/livepatch.txt +++ b/Documentation/livepatch/livepatch.txt | |||
@@ -72,7 +72,8 @@ example, they add a NULL pointer or a boundary check, fix a race by adding | |||
72 | a missing memory barrier, or add some locking around a critical section. | 72 | a missing memory barrier, or add some locking around a critical section. |
73 | Most of these changes are self contained and the function presents itself | 73 | Most of these changes are self contained and the function presents itself |
74 | the same way to the rest of the system. In this case, the functions might | 74 | the same way to the rest of the system. In this case, the functions might |
75 | be updated independently one by one. | 75 | be updated independently one by one. (This can be done by setting the |
76 | 'immediate' flag in the klp_patch struct.) | ||
76 | 77 | ||
77 | But there are more complex fixes. For example, a patch might change | 78 | But there are more complex fixes. For example, a patch might change |
78 | ordering of locking in multiple functions at the same time. Or a patch | 79 | ordering of locking in multiple functions at the same time. Or a patch |
@@ -86,20 +87,141 @@ or no data are stored in the modified structures at the moment. | |||
86 | The theory about how to apply functions a safe way is rather complex. | 87 | The theory about how to apply functions a safe way is rather complex. |
87 | The aim is to define a so-called consistency model. It attempts to define | 88 | The aim is to define a so-called consistency model. It attempts to define |
88 | conditions when the new implementation could be used so that the system | 89 | conditions when the new implementation could be used so that the system |
89 | stays consistent. The theory is not yet finished. See the discussion at | 90 | stays consistent. |
90 | https://lkml.kernel.org/r/20141107140458.GA21774@suse.cz | 91 | |
91 | 92 | Livepatch has a consistency model which is a hybrid of kGraft and | |
92 | The current consistency model is very simple. It guarantees that either | 93 | kpatch: it uses kGraft's per-task consistency and syscall barrier |
93 | the old or the new function is called. But various functions get redirected | 94 | switching combined with kpatch's stack trace switching. There are also |
94 | one by one without any synchronization. | 95 | a number of fallback options which make it quite flexible. |
95 | 96 | ||
96 | In other words, the current implementation _never_ modifies the behavior | 97 | Patches are applied on a per-task basis, when the task is deemed safe to |
97 | in the middle of the call. It is because it does _not_ rewrite the entire | 98 | switch over. When a patch is enabled, livepatch enters into a |
98 | function in the memory. Instead, the function gets redirected at the | 99 | transition state where tasks are converging to the patched state. |
99 | very beginning. But this redirection is used immediately even when | 100 | Usually this transition state can complete in a few seconds. The same |
100 | some other functions from the same patch have not been redirected yet. | 101 | sequence occurs when a patch is disabled, except the tasks converge from |
101 | 102 | the patched state to the unpatched state. | |
102 | See also the section "Limitations" below. | 103 | |
104 | An interrupt handler inherits the patched state of the task it | ||
105 | interrupts. The same is true for forked tasks: the child inherits the | ||
106 | patched state of the parent. | ||
107 | |||
108 | Livepatch uses several complementary approaches to determine when it's | ||
109 | safe to patch tasks: | ||
110 | |||
111 | 1. The first and most effective approach is stack checking of sleeping | ||
112 | tasks. If no affected functions are on the stack of a given task, | ||
113 | the task is patched. In most cases this will patch most or all of | ||
114 | the tasks on the first try. Otherwise it'll keep trying | ||
115 | periodically. This option is only available if the architecture has | ||
116 | reliable stacks (HAVE_RELIABLE_STACKTRACE). | ||
117 | |||
118 | 2. The second approach, if needed, is kernel exit switching. A | ||
119 | task is switched when it returns to user space from a system call, a | ||
120 | user space IRQ, or a signal. It's useful in the following cases: | ||
121 | |||
122 | a) Patching I/O-bound user tasks which are sleeping on an affected | ||
123 | function. In this case you have to send SIGSTOP and SIGCONT to | ||
124 | force it to exit the kernel and be patched. | ||
125 | b) Patching CPU-bound user tasks. If the task is highly CPU-bound | ||
126 | then it will get patched the next time it gets interrupted by an | ||
127 | IRQ. | ||
128 | c) In the future it could be useful for applying patches for | ||
129 | architectures which don't yet have HAVE_RELIABLE_STACKTRACE. In | ||
130 | this case you would have to signal most of the tasks on the | ||
131 | system. However this isn't supported yet because there's | ||
132 | currently no way to patch kthreads without | ||
133 | HAVE_RELIABLE_STACKTRACE. | ||
134 | |||
135 | 3. For idle "swapper" tasks, since they don't ever exit the kernel, they | ||
136 | instead have a klp_update_patch_state() call in the idle loop which | ||
137 | allows them to be patched before the CPU enters the idle state. | ||
138 | |||
139 | (Note there's not yet such an approach for kthreads.) | ||
140 | |||
141 | All the above approaches may be skipped by setting the 'immediate' flag | ||
142 | in the 'klp_patch' struct, which will disable per-task consistency and | ||
143 | patch all tasks immediately. This can be useful if the patch doesn't | ||
144 | change any function or data semantics. Note that, even with this flag | ||
145 | set, it's possible that some tasks may still be running with an old | ||
146 | version of the function, until that function returns. | ||
147 | |||
148 | There's also an 'immediate' flag in the 'klp_func' struct which allows | ||
149 | you to specify that certain functions in the patch can be applied | ||
150 | without per-task consistency. This might be useful if you want to patch | ||
151 | a common function like schedule(), and the function change doesn't need | ||
152 | consistency but the rest of the patch does. | ||
153 | |||
154 | For architectures which don't have HAVE_RELIABLE_STACKTRACE, the user | ||
155 | must set patch->immediate which causes all tasks to be patched | ||
156 | immediately. This option should be used with care, only when the patch | ||
157 | doesn't change any function or data semantics. | ||
158 | |||
159 | In the future, architectures which don't have HAVE_RELIABLE_STACKTRACE | ||
160 | may be allowed to use per-task consistency if we can come up with | ||
161 | another way to patch kthreads. | ||
162 | |||
163 | The /sys/kernel/livepatch/<patch>/transition file shows whether a patch | ||
164 | is in transition. Only a single patch (the topmost patch on the stack) | ||
165 | can be in transition at a given time. A patch can remain in transition | ||
166 | indefinitely, if any of the tasks are stuck in the initial patch state. | ||
167 | |||
168 | A transition can be reversed and effectively canceled by writing the | ||
169 | opposite value to the /sys/kernel/livepatch/<patch>/enabled file while | ||
170 | the transition is in progress. Then all the tasks will attempt to | ||
171 | converge back to the original patch state. | ||
172 | |||
173 | There's also a /proc/<pid>/patch_state file which can be used to | ||
174 | determine which tasks are blocking completion of a patching operation. | ||
175 | If a patch is in transition, this file shows 0 to indicate the task is | ||
176 | unpatched and 1 to indicate it's patched. Otherwise, if no patch is in | ||
177 | transition, it shows -1. Any tasks which are blocking the transition | ||
178 | can be signaled with SIGSTOP and SIGCONT to force them to change their | ||
179 | patched state. | ||
180 | |||
181 | |||
182 | 3.1 Adding consistency model support to new architectures | ||
183 | --------------------------------------------------------- | ||
184 | |||
185 | For adding consistency model support to new architectures, there are a | ||
186 | few options: | ||
187 | |||
188 | 1) Add CONFIG_HAVE_RELIABLE_STACKTRACE. This means porting objtool, and | ||
189 | for non-DWARF unwinders, also making sure there's a way for the stack | ||
190 | tracing code to detect interrupts on the stack. | ||
191 | |||
192 | 2) Alternatively, ensure that every kthread has a call to | ||
193 | klp_update_patch_state() in a safe location. Kthreads are typically | ||
194 | in an infinite loop which does some action repeatedly. The safe | ||
195 | location to switch the kthread's patch state would be at a designated | ||
196 | point in the loop where there are no locks taken and all data | ||
197 | structures are in a well-defined state. | ||
198 | |||
199 | The location is clear when using workqueues or the kthread worker | ||
200 | API. These kthreads process independent actions in a generic loop. | ||
201 | |||
202 | It's much more complicated with kthreads which have a custom loop. | ||
203 | There the safe location must be carefully selected on a case-by-case | ||
204 | basis. | ||
205 | |||
206 | In that case, arches without HAVE_RELIABLE_STACKTRACE would still be | ||
207 | able to use the non-stack-checking parts of the consistency model: | ||
208 | |||
209 | a) patching user tasks when they cross the kernel/user space | ||
210 | boundary; and | ||
211 | |||
212 | b) patching kthreads and idle tasks at their designated patch points. | ||
213 | |||
214 | This option isn't as good as option 1 because it requires signaling | ||
215 | user tasks and waking kthreads to patch them. But it could still be | ||
216 | a good backup option for those architectures which don't have | ||
217 | reliable stack traces yet. | ||
218 | |||
219 | In the meantime, patches for such architectures can bypass the | ||
220 | consistency model by setting klp_patch.immediate to true. This option | ||
221 | is perfectly fine for patches which don't change the semantics of the | ||
222 | patched functions. In practice, this is usable for ~90% of security | ||
223 | fixes. Use of this option also means the patch can't be unloaded after | ||
224 | it has been disabled. | ||
103 | 225 | ||
104 | 226 | ||
105 | 4. Livepatch module | 227 | 4. Livepatch module |
@@ -134,7 +256,7 @@ Documentation/livepatch/module-elf-format.txt for more details. | |||
134 | 256 | ||
135 | 257 | ||
136 | 4.2. Metadata | 258 | 4.2. Metadata |
137 | ------------ | 259 | ------------- |
138 | 260 | ||
139 | The patch is described by several structures that split the information | 261 | The patch is described by several structures that split the information |
140 | into three levels: | 262 | into three levels: |
@@ -156,6 +278,9 @@ into three levels: | |||
156 | only for a particular object ( vmlinux or a kernel module ). Note that | 278 | only for a particular object ( vmlinux or a kernel module ). Note that |
157 | kallsyms allows for searching symbols according to the object name. | 279 | kallsyms allows for searching symbols according to the object name. |
158 | 280 | ||
281 | There's also an 'immediate' flag which, when set, patches the | ||
282 | function immediately, bypassing the consistency model safety checks. | ||
283 | |||
159 | + struct klp_object defines an array of patched functions (struct | 284 | + struct klp_object defines an array of patched functions (struct |
160 | klp_func) in the same object. Where the object is either vmlinux | 285 | klp_func) in the same object. Where the object is either vmlinux |
161 | (NULL) or a module name. | 286 | (NULL) or a module name. |
@@ -172,10 +297,13 @@ into three levels: | |||
172 | This structure handles all patched functions consistently and eventually, | 297 | This structure handles all patched functions consistently and eventually, |
173 | synchronously. The whole patch is applied only when all patched | 298 | synchronously. The whole patch is applied only when all patched |
174 | symbols are found. The only exception are symbols from objects | 299 | symbols are found. The only exception are symbols from objects |
175 | (kernel modules) that have not been loaded yet. Also if a more complex | 300 | (kernel modules) that have not been loaded yet. |
176 | consistency model is supported then a selected unit (thread, | 301 | |
177 | kernel as a whole) will see the new code from the entire patch | 302 | Setting the 'immediate' flag applies the patch to all tasks |
178 | only when it is in a safe state. | 303 | immediately, bypassing the consistency model safety checks. |
304 | |||
305 | For more details on how the patch is applied on a per-task basis, | ||
306 | see the "Consistency model" section. | ||
179 | 307 | ||
180 | 308 | ||
181 | 4.3. Livepatch module handling | 309 | 4.3. Livepatch module handling |
@@ -239,9 +367,15 @@ Registered patches might be enabled either by calling klp_enable_patch() or | |||
239 | by writing '1' to /sys/kernel/livepatch/<name>/enabled. The system will | 367 | by writing '1' to /sys/kernel/livepatch/<name>/enabled. The system will |
240 | start using the new implementation of the patched functions at this stage. | 368 | start using the new implementation of the patched functions at this stage. |
241 | 369 | ||
242 | In particular, if an original function is patched for the first time, a | 370 | When a patch is enabled, livepatch enters into a transition state where |
243 | function specific struct klp_ops is created and an universal ftrace handler | 371 | tasks are converging to the patched state. This is indicated by a value |
244 | is registered. | 372 | of '1' in /sys/kernel/livepatch/<name>/transition. Once all tasks have |
373 | been patched, the 'transition' value changes to '0'. For more | ||
374 | information about this process, see the "Consistency model" section. | ||
375 | |||
376 | If an original function is patched for the first time, a function | ||
377 | specific struct klp_ops is created and an universal ftrace handler is | ||
378 | registered. | ||
245 | 379 | ||
246 | Functions might be patched multiple times. The ftrace handler is registered | 380 | Functions might be patched multiple times. The ftrace handler is registered |
247 | only once for the given function. Further patches just add an entry to the | 381 | only once for the given function. Further patches just add an entry to the |
@@ -261,6 +395,12 @@ by writing '0' to /sys/kernel/livepatch/<name>/enabled. At this stage | |||
261 | either the code from the previously enabled patch or even the original | 395 | either the code from the previously enabled patch or even the original |
262 | code gets used. | 396 | code gets used. |
263 | 397 | ||
398 | When a patch is disabled, livepatch enters into a transition state where | ||
399 | tasks are converging to the unpatched state. This is indicated by a | ||
400 | value of '1' in /sys/kernel/livepatch/<name>/transition. Once all tasks | ||
401 | have been unpatched, the 'transition' value changes to '0'. For more | ||
402 | information about this process, see the "Consistency model" section. | ||
403 | |||
264 | Here all the functions (struct klp_func) associated with the to-be-disabled | 404 | Here all the functions (struct klp_func) associated with the to-be-disabled |
265 | patch are removed from the corresponding struct klp_ops. The ftrace handler | 405 | patch are removed from the corresponding struct klp_ops. The ftrace handler |
266 | is unregistered and the struct klp_ops is freed when the func_stack list | 406 | is unregistered and the struct klp_ops is freed when the func_stack list |
diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 91d9049f0039..5a791055b176 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h | |||
@@ -15,6 +15,7 @@ | |||
15 | #include <linux/sched/autogroup.h> | 15 | #include <linux/sched/autogroup.h> |
16 | #include <net/net_namespace.h> | 16 | #include <net/net_namespace.h> |
17 | #include <linux/sched/rt.h> | 17 | #include <linux/sched/rt.h> |
18 | #include <linux/livepatch.h> | ||
18 | #include <linux/mm_types.h> | 19 | #include <linux/mm_types.h> |
19 | 20 | ||
20 | #include <asm/thread_info.h> | 21 | #include <asm/thread_info.h> |
@@ -202,6 +203,13 @@ extern struct cred init_cred; | |||
202 | # define INIT_KASAN(tsk) | 203 | # define INIT_KASAN(tsk) |
203 | #endif | 204 | #endif |
204 | 205 | ||
206 | #ifdef CONFIG_LIVEPATCH | ||
207 | # define INIT_LIVEPATCH(tsk) \ | ||
208 | .patch_state = KLP_UNDEFINED, | ||
209 | #else | ||
210 | # define INIT_LIVEPATCH(tsk) | ||
211 | #endif | ||
212 | |||
205 | #ifdef CONFIG_THREAD_INFO_IN_TASK | 213 | #ifdef CONFIG_THREAD_INFO_IN_TASK |
206 | # define INIT_TASK_TI(tsk) \ | 214 | # define INIT_TASK_TI(tsk) \ |
207 | .thread_info = INIT_THREAD_INFO(tsk), \ | 215 | .thread_info = INIT_THREAD_INFO(tsk), \ |
@@ -288,6 +296,7 @@ extern struct cred init_cred; | |||
288 | INIT_VTIME(tsk) \ | 296 | INIT_VTIME(tsk) \ |
289 | INIT_NUMA_BALANCING(tsk) \ | 297 | INIT_NUMA_BALANCING(tsk) \ |
290 | INIT_KASAN(tsk) \ | 298 | INIT_KASAN(tsk) \ |
299 | INIT_LIVEPATCH(tsk) \ | ||
291 | } | 300 | } |
292 | 301 | ||
293 | 302 | ||
diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 6602b34bed2b..ed90ad1605c1 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h | |||
@@ -28,18 +28,40 @@ | |||
28 | 28 | ||
29 | #include <asm/livepatch.h> | 29 | #include <asm/livepatch.h> |
30 | 30 | ||
31 | /* task patch states */ | ||
32 | #define KLP_UNDEFINED -1 | ||
33 | #define KLP_UNPATCHED 0 | ||
34 | #define KLP_PATCHED 1 | ||
35 | |||
31 | /** | 36 | /** |
32 | * struct klp_func - function structure for live patching | 37 | * struct klp_func - function structure for live patching |
33 | * @old_name: name of the function to be patched | 38 | * @old_name: name of the function to be patched |
34 | * @new_func: pointer to the patched function code | 39 | * @new_func: pointer to the patched function code |
35 | * @old_sympos: a hint indicating which symbol position the old function | 40 | * @old_sympos: a hint indicating which symbol position the old function |
36 | * can be found (optional) | 41 | * can be found (optional) |
42 | * @immediate: patch the func immediately, bypassing safety mechanisms | ||
37 | * @old_addr: the address of the function being patched | 43 | * @old_addr: the address of the function being patched |
38 | * @kobj: kobject for sysfs resources | 44 | * @kobj: kobject for sysfs resources |
39 | * @stack_node: list node for klp_ops func_stack list | 45 | * @stack_node: list node for klp_ops func_stack list |
40 | * @old_size: size of the old function | 46 | * @old_size: size of the old function |
41 | * @new_size: size of the new function | 47 | * @new_size: size of the new function |
42 | * @patched: the func has been added to the klp_ops list | 48 | * @patched: the func has been added to the klp_ops list |
49 | * @transition: the func is currently being applied or reverted | ||
50 | * | ||
51 | * The patched and transition variables define the func's patching state. When | ||
52 | * patching, a func is always in one of the following states: | ||
53 | * | ||
54 | * patched=0 transition=0: unpatched | ||
55 | * patched=0 transition=1: unpatched, temporary starting state | ||
56 | * patched=1 transition=1: patched, may be visible to some tasks | ||
57 | * patched=1 transition=0: patched, visible to all tasks | ||
58 | * | ||
59 | * And when unpatching, it goes in the reverse order: | ||
60 | * | ||
61 | * patched=1 transition=0: patched, visible to all tasks | ||
62 | * patched=1 transition=1: patched, may be visible to some tasks | ||
63 | * patched=0 transition=1: unpatched, temporary ending state | ||
64 | * patched=0 transition=0: unpatched | ||
43 | */ | 65 | */ |
44 | struct klp_func { | 66 | struct klp_func { |
45 | /* external */ | 67 | /* external */ |
@@ -53,6 +75,7 @@ struct klp_func { | |||
53 | * in kallsyms for the given object is used. | 75 | * in kallsyms for the given object is used. |
54 | */ | 76 | */ |
55 | unsigned long old_sympos; | 77 | unsigned long old_sympos; |
78 | bool immediate; | ||
56 | 79 | ||
57 | /* internal */ | 80 | /* internal */ |
58 | unsigned long old_addr; | 81 | unsigned long old_addr; |
@@ -60,6 +83,7 @@ struct klp_func { | |||
60 | struct list_head stack_node; | 83 | struct list_head stack_node; |
61 | unsigned long old_size, new_size; | 84 | unsigned long old_size, new_size; |
62 | bool patched; | 85 | bool patched; |
86 | bool transition; | ||
63 | }; | 87 | }; |
64 | 88 | ||
65 | /** | 89 | /** |
@@ -68,7 +92,7 @@ struct klp_func { | |||
68 | * @funcs: function entries for functions to be patched in the object | 92 | * @funcs: function entries for functions to be patched in the object |
69 | * @kobj: kobject for sysfs resources | 93 | * @kobj: kobject for sysfs resources |
70 | * @mod: kernel module associated with the patched object | 94 | * @mod: kernel module associated with the patched object |
71 | * (NULL for vmlinux) | 95 | * (NULL for vmlinux) |
72 | * @patched: the object's funcs have been added to the klp_ops list | 96 | * @patched: the object's funcs have been added to the klp_ops list |
73 | */ | 97 | */ |
74 | struct klp_object { | 98 | struct klp_object { |
@@ -86,6 +110,7 @@ struct klp_object { | |||
86 | * struct klp_patch - patch structure for live patching | 110 | * struct klp_patch - patch structure for live patching |
87 | * @mod: reference to the live patch module | 111 | * @mod: reference to the live patch module |
88 | * @objs: object entries for kernel objects to be patched | 112 | * @objs: object entries for kernel objects to be patched |
113 | * @immediate: patch all funcs immediately, bypassing safety mechanisms | ||
89 | * @list: list node for global list of registered patches | 114 | * @list: list node for global list of registered patches |
90 | * @kobj: kobject for sysfs resources | 115 | * @kobj: kobject for sysfs resources |
91 | * @enabled: the patch is enabled (but operation may be incomplete) | 116 | * @enabled: the patch is enabled (but operation may be incomplete) |
@@ -94,6 +119,7 @@ struct klp_patch { | |||
94 | /* external */ | 119 | /* external */ |
95 | struct module *mod; | 120 | struct module *mod; |
96 | struct klp_object *objs; | 121 | struct klp_object *objs; |
122 | bool immediate; | ||
97 | 123 | ||
98 | /* internal */ | 124 | /* internal */ |
99 | struct list_head list; | 125 | struct list_head list; |
@@ -121,13 +147,27 @@ void arch_klp_init_object_loaded(struct klp_patch *patch, | |||
121 | int klp_module_coming(struct module *mod); | 147 | int klp_module_coming(struct module *mod); |
122 | void klp_module_going(struct module *mod); | 148 | void klp_module_going(struct module *mod); |
123 | 149 | ||
150 | void klp_copy_process(struct task_struct *child); | ||
124 | void klp_update_patch_state(struct task_struct *task); | 151 | void klp_update_patch_state(struct task_struct *task); |
125 | 152 | ||
153 | static inline bool klp_patch_pending(struct task_struct *task) | ||
154 | { | ||
155 | return test_tsk_thread_flag(task, TIF_PATCH_PENDING); | ||
156 | } | ||
157 | |||
158 | static inline bool klp_have_reliable_stack(void) | ||
159 | { | ||
160 | return IS_ENABLED(CONFIG_STACKTRACE) && | ||
161 | IS_ENABLED(CONFIG_HAVE_RELIABLE_STACKTRACE); | ||
162 | } | ||
163 | |||
126 | #else /* !CONFIG_LIVEPATCH */ | 164 | #else /* !CONFIG_LIVEPATCH */ |
127 | 165 | ||
128 | static inline int klp_module_coming(struct module *mod) { return 0; } | 166 | static inline int klp_module_coming(struct module *mod) { return 0; } |
129 | static inline void klp_module_going(struct module *mod) {} | 167 | static inline void klp_module_going(struct module *mod) {} |
168 | static inline bool klp_patch_pending(struct task_struct *task) { return false; } | ||
130 | static inline void klp_update_patch_state(struct task_struct *task) {} | 169 | static inline void klp_update_patch_state(struct task_struct *task) {} |
170 | static inline void klp_copy_process(struct task_struct *child) {} | ||
131 | 171 | ||
132 | #endif /* CONFIG_LIVEPATCH */ | 172 | #endif /* CONFIG_LIVEPATCH */ |
133 | 173 | ||
diff --git a/include/linux/sched.h b/include/linux/sched.h index d67eee84fd43..e11032010318 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -1038,6 +1038,9 @@ struct task_struct { | |||
1038 | /* A live task holds one reference: */ | 1038 | /* A live task holds one reference: */ |
1039 | atomic_t stack_refcount; | 1039 | atomic_t stack_refcount; |
1040 | #endif | 1040 | #endif |
1041 | #ifdef CONFIG_LIVEPATCH | ||
1042 | int patch_state; | ||
1043 | #endif | ||
1041 | /* CPU-specific state of this task: */ | 1044 | /* CPU-specific state of this task: */ |
1042 | struct thread_struct thread; | 1045 | struct thread_struct thread; |
1043 | 1046 | ||
diff --git a/kernel/fork.c b/kernel/fork.c index 6c463c80e93d..942cbcd07c18 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -87,6 +87,7 @@ | |||
87 | #include <linux/compiler.h> | 87 | #include <linux/compiler.h> |
88 | #include <linux/sysctl.h> | 88 | #include <linux/sysctl.h> |
89 | #include <linux/kcov.h> | 89 | #include <linux/kcov.h> |
90 | #include <linux/livepatch.h> | ||
90 | 91 | ||
91 | #include <asm/pgtable.h> | 92 | #include <asm/pgtable.h> |
92 | #include <asm/pgalloc.h> | 93 | #include <asm/pgalloc.h> |
@@ -1797,6 +1798,8 @@ static __latent_entropy struct task_struct *copy_process( | |||
1797 | p->parent_exec_id = current->self_exec_id; | 1798 | p->parent_exec_id = current->self_exec_id; |
1798 | } | 1799 | } |
1799 | 1800 | ||
1801 | klp_copy_process(p); | ||
1802 | |||
1800 | spin_lock(¤t->sighand->siglock); | 1803 | spin_lock(¤t->sighand->siglock); |
1801 | 1804 | ||
1802 | /* | 1805 | /* |
diff --git a/kernel/livepatch/Makefile b/kernel/livepatch/Makefile index e136dad8ff7e..2b8bdb1925da 100644 --- a/kernel/livepatch/Makefile +++ b/kernel/livepatch/Makefile | |||
@@ -1,3 +1,3 @@ | |||
1 | obj-$(CONFIG_LIVEPATCH) += livepatch.o | 1 | obj-$(CONFIG_LIVEPATCH) += livepatch.o |
2 | 2 | ||
3 | livepatch-objs := core.o patch.o | 3 | livepatch-objs := core.o patch.o transition.o |
diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 10ba3a1578bd..3dc3c9049690 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c | |||
@@ -31,22 +31,22 @@ | |||
31 | #include <linux/moduleloader.h> | 31 | #include <linux/moduleloader.h> |
32 | #include <asm/cacheflush.h> | 32 | #include <asm/cacheflush.h> |
33 | #include "patch.h" | 33 | #include "patch.h" |
34 | #include "transition.h" | ||
34 | 35 | ||
35 | /* | 36 | /* |
36 | * The klp_mutex protects the global lists and state transitions of any | 37 | * klp_mutex is a coarse lock which serializes access to klp data. All |
37 | * structure reachable from them. References to any structure must be obtained | 38 | * accesses to klp-related variables and structures must have mutex protection, |
38 | * under mutex protection (except in klp_ftrace_handler(), which uses RCU to | 39 | * except within the following functions which carefully avoid the need for it: |
39 | * ensure it gets consistent data). | 40 | * |
41 | * - klp_ftrace_handler() | ||
42 | * - klp_update_patch_state() | ||
40 | */ | 43 | */ |
41 | static DEFINE_MUTEX(klp_mutex); | 44 | DEFINE_MUTEX(klp_mutex); |
42 | 45 | ||
43 | static LIST_HEAD(klp_patches); | 46 | static LIST_HEAD(klp_patches); |
44 | 47 | ||
45 | static struct kobject *klp_root_kobj; | 48 | static struct kobject *klp_root_kobj; |
46 | 49 | ||
47 | /* TODO: temporary stub */ | ||
48 | void klp_update_patch_state(struct task_struct *task) {} | ||
49 | |||
50 | static bool klp_is_module(struct klp_object *obj) | 50 | static bool klp_is_module(struct klp_object *obj) |
51 | { | 51 | { |
52 | return obj->name; | 52 | return obj->name; |
@@ -85,7 +85,6 @@ static void klp_find_object_module(struct klp_object *obj) | |||
85 | mutex_unlock(&module_mutex); | 85 | mutex_unlock(&module_mutex); |
86 | } | 86 | } |
87 | 87 | ||
88 | /* klp_mutex must be held by caller */ | ||
89 | static bool klp_is_patch_registered(struct klp_patch *patch) | 88 | static bool klp_is_patch_registered(struct klp_patch *patch) |
90 | { | 89 | { |
91 | struct klp_patch *mypatch; | 90 | struct klp_patch *mypatch; |
@@ -281,20 +280,27 @@ static int klp_write_object_relocations(struct module *pmod, | |||
281 | 280 | ||
282 | static int __klp_disable_patch(struct klp_patch *patch) | 281 | static int __klp_disable_patch(struct klp_patch *patch) |
283 | { | 282 | { |
284 | struct klp_object *obj; | 283 | if (klp_transition_patch) |
284 | return -EBUSY; | ||
285 | 285 | ||
286 | /* enforce stacking: only the last enabled patch can be disabled */ | 286 | /* enforce stacking: only the last enabled patch can be disabled */ |
287 | if (!list_is_last(&patch->list, &klp_patches) && | 287 | if (!list_is_last(&patch->list, &klp_patches) && |
288 | list_next_entry(patch, list)->enabled) | 288 | list_next_entry(patch, list)->enabled) |
289 | return -EBUSY; | 289 | return -EBUSY; |
290 | 290 | ||
291 | pr_notice("disabling patch '%s'\n", patch->mod->name); | 291 | klp_init_transition(patch, KLP_UNPATCHED); |
292 | 292 | ||
293 | klp_for_each_object(patch, obj) { | 293 | /* |
294 | if (obj->patched) | 294 | * Enforce the order of the func->transition writes in |
295 | klp_unpatch_object(obj); | 295 | * klp_init_transition() and the TIF_PATCH_PENDING writes in |
296 | } | 296 | * klp_start_transition(). In the rare case where klp_ftrace_handler() |
297 | * is called shortly after klp_update_patch_state() switches the task, | ||
298 | * this ensures the handler sees that func->transition is set. | ||
299 | */ | ||
300 | smp_wmb(); | ||
297 | 301 | ||
302 | klp_start_transition(); | ||
303 | klp_try_complete_transition(); | ||
298 | patch->enabled = false; | 304 | patch->enabled = false; |
299 | 305 | ||
300 | return 0; | 306 | return 0; |
@@ -337,6 +343,9 @@ static int __klp_enable_patch(struct klp_patch *patch) | |||
337 | struct klp_object *obj; | 343 | struct klp_object *obj; |
338 | int ret; | 344 | int ret; |
339 | 345 | ||
346 | if (klp_transition_patch) | ||
347 | return -EBUSY; | ||
348 | |||
340 | if (WARN_ON(patch->enabled)) | 349 | if (WARN_ON(patch->enabled)) |
341 | return -EINVAL; | 350 | return -EINVAL; |
342 | 351 | ||
@@ -347,22 +356,36 @@ static int __klp_enable_patch(struct klp_patch *patch) | |||
347 | 356 | ||
348 | pr_notice("enabling patch '%s'\n", patch->mod->name); | 357 | pr_notice("enabling patch '%s'\n", patch->mod->name); |
349 | 358 | ||
359 | klp_init_transition(patch, KLP_PATCHED); | ||
360 | |||
361 | /* | ||
362 | * Enforce the order of the func->transition writes in | ||
363 | * klp_init_transition() and the ops->func_stack writes in | ||
364 | * klp_patch_object(), so that klp_ftrace_handler() will see the | ||
365 | * func->transition updates before the handler is registered and the | ||
366 | * new funcs become visible to the handler. | ||
367 | */ | ||
368 | smp_wmb(); | ||
369 | |||
350 | klp_for_each_object(patch, obj) { | 370 | klp_for_each_object(patch, obj) { |
351 | if (!klp_is_object_loaded(obj)) | 371 | if (!klp_is_object_loaded(obj)) |
352 | continue; | 372 | continue; |
353 | 373 | ||
354 | ret = klp_patch_object(obj); | 374 | ret = klp_patch_object(obj); |
355 | if (ret) | 375 | if (ret) { |
356 | goto unregister; | 376 | pr_warn("failed to enable patch '%s'\n", |
377 | patch->mod->name); | ||
378 | |||
379 | klp_cancel_transition(); | ||
380 | return ret; | ||
381 | } | ||
357 | } | 382 | } |
358 | 383 | ||
384 | klp_start_transition(); | ||
385 | klp_try_complete_transition(); | ||
359 | patch->enabled = true; | 386 | patch->enabled = true; |
360 | 387 | ||
361 | return 0; | 388 | return 0; |
362 | |||
363 | unregister: | ||
364 | WARN_ON(__klp_disable_patch(patch)); | ||
365 | return ret; | ||
366 | } | 389 | } |
367 | 390 | ||
368 | /** | 391 | /** |
@@ -399,6 +422,7 @@ EXPORT_SYMBOL_GPL(klp_enable_patch); | |||
399 | * /sys/kernel/livepatch | 422 | * /sys/kernel/livepatch |
400 | * /sys/kernel/livepatch/<patch> | 423 | * /sys/kernel/livepatch/<patch> |
401 | * /sys/kernel/livepatch/<patch>/enabled | 424 | * /sys/kernel/livepatch/<patch>/enabled |
425 | * /sys/kernel/livepatch/<patch>/transition | ||
402 | * /sys/kernel/livepatch/<patch>/<object> | 426 | * /sys/kernel/livepatch/<patch>/<object> |
403 | * /sys/kernel/livepatch/<patch>/<object>/<function,sympos> | 427 | * /sys/kernel/livepatch/<patch>/<object>/<function,sympos> |
404 | */ | 428 | */ |
@@ -424,7 +448,9 @@ static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, | |||
424 | goto err; | 448 | goto err; |
425 | } | 449 | } |
426 | 450 | ||
427 | if (enabled) { | 451 | if (patch == klp_transition_patch) { |
452 | klp_reverse_transition(); | ||
453 | } else if (enabled) { | ||
428 | ret = __klp_enable_patch(patch); | 454 | ret = __klp_enable_patch(patch); |
429 | if (ret) | 455 | if (ret) |
430 | goto err; | 456 | goto err; |
@@ -452,9 +478,21 @@ static ssize_t enabled_show(struct kobject *kobj, | |||
452 | return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled); | 478 | return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled); |
453 | } | 479 | } |
454 | 480 | ||
481 | static ssize_t transition_show(struct kobject *kobj, | ||
482 | struct kobj_attribute *attr, char *buf) | ||
483 | { | ||
484 | struct klp_patch *patch; | ||
485 | |||
486 | patch = container_of(kobj, struct klp_patch, kobj); | ||
487 | return snprintf(buf, PAGE_SIZE-1, "%d\n", | ||
488 | patch == klp_transition_patch); | ||
489 | } | ||
490 | |||
455 | static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled); | 491 | static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled); |
492 | static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition); | ||
456 | static struct attribute *klp_patch_attrs[] = { | 493 | static struct attribute *klp_patch_attrs[] = { |
457 | &enabled_kobj_attr.attr, | 494 | &enabled_kobj_attr.attr, |
495 | &transition_kobj_attr.attr, | ||
458 | NULL | 496 | NULL |
459 | }; | 497 | }; |
460 | 498 | ||
@@ -544,6 +582,7 @@ static int klp_init_func(struct klp_object *obj, struct klp_func *func) | |||
544 | 582 | ||
545 | INIT_LIST_HEAD(&func->stack_node); | 583 | INIT_LIST_HEAD(&func->stack_node); |
546 | func->patched = false; | 584 | func->patched = false; |
585 | func->transition = false; | ||
547 | 586 | ||
548 | /* The format for the sysfs directory is <function,sympos> where sympos | 587 | /* The format for the sysfs directory is <function,sympos> where sympos |
549 | * is the nth occurrence of this symbol in kallsyms for the patched | 588 | * is the nth occurrence of this symbol in kallsyms for the patched |
@@ -740,6 +779,16 @@ int klp_register_patch(struct klp_patch *patch) | |||
740 | return -ENODEV; | 779 | return -ENODEV; |
741 | 780 | ||
742 | /* | 781 | /* |
782 | * Architectures without reliable stack traces have to set | ||
783 | * patch->immediate because there's currently no way to patch kthreads | ||
784 | * with the consistency model. | ||
785 | */ | ||
786 | if (!klp_have_reliable_stack() && !patch->immediate) { | ||
787 | pr_err("This architecture doesn't have support for the livepatch consistency model.\n"); | ||
788 | return -ENOSYS; | ||
789 | } | ||
790 | |||
791 | /* | ||
743 | * A reference is taken on the patch module to prevent it from being | 792 | * A reference is taken on the patch module to prevent it from being |
744 | * unloaded. Right now, we don't allow patch modules to unload since | 793 | * unloaded. Right now, we don't allow patch modules to unload since |
745 | * there is currently no method to determine if a thread is still | 794 | * there is currently no method to determine if a thread is still |
@@ -788,7 +837,11 @@ int klp_module_coming(struct module *mod) | |||
788 | goto err; | 837 | goto err; |
789 | } | 838 | } |
790 | 839 | ||
791 | if (!patch->enabled) | 840 | /* |
841 | * Only patch the module if the patch is enabled or is | ||
842 | * in transition. | ||
843 | */ | ||
844 | if (!patch->enabled && patch != klp_transition_patch) | ||
792 | break; | 845 | break; |
793 | 846 | ||
794 | pr_notice("applying patch '%s' to loading module '%s'\n", | 847 | pr_notice("applying patch '%s' to loading module '%s'\n", |
@@ -845,7 +898,11 @@ void klp_module_going(struct module *mod) | |||
845 | if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) | 898 | if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) |
846 | continue; | 899 | continue; |
847 | 900 | ||
848 | if (patch->enabled) { | 901 | /* |
902 | * Only unpatch the module if the patch is enabled or | ||
903 | * is in transition. | ||
904 | */ | ||
905 | if (patch->enabled || patch == klp_transition_patch) { | ||
849 | pr_notice("reverting patch '%s' on unloading module '%s'\n", | 906 | pr_notice("reverting patch '%s' on unloading module '%s'\n", |
850 | patch->mod->name, obj->mod->name); | 907 | patch->mod->name, obj->mod->name); |
851 | klp_unpatch_object(obj); | 908 | klp_unpatch_object(obj); |
diff --git a/kernel/livepatch/patch.c b/kernel/livepatch/patch.c index 5efa2620851a..f8269036bf0b 100644 --- a/kernel/livepatch/patch.c +++ b/kernel/livepatch/patch.c | |||
@@ -29,6 +29,7 @@ | |||
29 | #include <linux/bug.h> | 29 | #include <linux/bug.h> |
30 | #include <linux/printk.h> | 30 | #include <linux/printk.h> |
31 | #include "patch.h" | 31 | #include "patch.h" |
32 | #include "transition.h" | ||
32 | 33 | ||
33 | static LIST_HEAD(klp_ops); | 34 | static LIST_HEAD(klp_ops); |
34 | 35 | ||
@@ -54,15 +55,64 @@ static void notrace klp_ftrace_handler(unsigned long ip, | |||
54 | { | 55 | { |
55 | struct klp_ops *ops; | 56 | struct klp_ops *ops; |
56 | struct klp_func *func; | 57 | struct klp_func *func; |
58 | int patch_state; | ||
57 | 59 | ||
58 | ops = container_of(fops, struct klp_ops, fops); | 60 | ops = container_of(fops, struct klp_ops, fops); |
59 | 61 | ||
60 | rcu_read_lock(); | 62 | rcu_read_lock(); |
63 | |||
61 | func = list_first_or_null_rcu(&ops->func_stack, struct klp_func, | 64 | func = list_first_or_null_rcu(&ops->func_stack, struct klp_func, |
62 | stack_node); | 65 | stack_node); |
66 | |||
67 | /* | ||
68 | * func should never be NULL because preemption should be disabled here | ||
69 | * and unregister_ftrace_function() does the equivalent of a | ||
70 | * synchronize_sched() before the func_stack removal. | ||
71 | */ | ||
63 | if (WARN_ON_ONCE(!func)) | 72 | if (WARN_ON_ONCE(!func)) |
64 | goto unlock; | 73 | goto unlock; |
65 | 74 | ||
75 | /* | ||
76 | * In the enable path, enforce the order of the ops->func_stack and | ||
77 | * func->transition reads. The corresponding write barrier is in | ||
78 | * __klp_enable_patch(). | ||
79 | * | ||
80 | * (Note that this barrier technically isn't needed in the disable | ||
81 | * path. In the rare case where klp_update_patch_state() runs before | ||
82 | * this handler, its TIF_PATCH_PENDING read and this func->transition | ||
83 | * read need to be ordered. But klp_update_patch_state() already | ||
84 | * enforces that.) | ||
85 | */ | ||
86 | smp_rmb(); | ||
87 | |||
88 | if (unlikely(func->transition)) { | ||
89 | |||
90 | /* | ||
91 | * Enforce the order of the func->transition and | ||
92 | * current->patch_state reads. Otherwise we could read an | ||
93 | * out-of-date task state and pick the wrong function. The | ||
94 | * corresponding write barrier is in klp_init_transition(). | ||
95 | */ | ||
96 | smp_rmb(); | ||
97 | |||
98 | patch_state = current->patch_state; | ||
99 | |||
100 | WARN_ON_ONCE(patch_state == KLP_UNDEFINED); | ||
101 | |||
102 | if (patch_state == KLP_UNPATCHED) { | ||
103 | /* | ||
104 | * Use the previously patched version of the function. | ||
105 | * If no previous patches exist, continue with the | ||
106 | * original function. | ||
107 | */ | ||
108 | func = list_entry_rcu(func->stack_node.next, | ||
109 | struct klp_func, stack_node); | ||
110 | |||
111 | if (&func->stack_node == &ops->func_stack) | ||
112 | goto unlock; | ||
113 | } | ||
114 | } | ||
115 | |||
66 | klp_arch_set_pc(regs, (unsigned long)func->new_func); | 116 | klp_arch_set_pc(regs, (unsigned long)func->new_func); |
67 | unlock: | 117 | unlock: |
68 | rcu_read_unlock(); | 118 | rcu_read_unlock(); |
@@ -211,3 +261,12 @@ int klp_patch_object(struct klp_object *obj) | |||
211 | 261 | ||
212 | return 0; | 262 | return 0; |
213 | } | 263 | } |
264 | |||
265 | void klp_unpatch_objects(struct klp_patch *patch) | ||
266 | { | ||
267 | struct klp_object *obj; | ||
268 | |||
269 | klp_for_each_object(patch, obj) | ||
270 | if (obj->patched) | ||
271 | klp_unpatch_object(obj); | ||
272 | } | ||
diff --git a/kernel/livepatch/patch.h b/kernel/livepatch/patch.h index 2d0cce02dade..0db227170c36 100644 --- a/kernel/livepatch/patch.h +++ b/kernel/livepatch/patch.h | |||
@@ -28,5 +28,6 @@ struct klp_ops *klp_find_ops(unsigned long old_addr); | |||
28 | 28 | ||
29 | int klp_patch_object(struct klp_object *obj); | 29 | int klp_patch_object(struct klp_object *obj); |
30 | void klp_unpatch_object(struct klp_object *obj); | 30 | void klp_unpatch_object(struct klp_object *obj); |
31 | void klp_unpatch_objects(struct klp_patch *patch); | ||
31 | 32 | ||
32 | #endif /* _LIVEPATCH_PATCH_H */ | 33 | #endif /* _LIVEPATCH_PATCH_H */ |
diff --git a/kernel/livepatch/transition.c b/kernel/livepatch/transition.c new file mode 100644 index 000000000000..428533ec51b5 --- /dev/null +++ b/kernel/livepatch/transition.c | |||
@@ -0,0 +1,543 @@ | |||
1 | /* | ||
2 | * transition.c - Kernel Live Patching transition functions | ||
3 | * | ||
4 | * Copyright (C) 2015-2016 Josh Poimboeuf <jpoimboe@redhat.com> | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or | ||
7 | * modify it under the terms of the GNU General Public License | ||
8 | * as published by the Free Software Foundation; either version 2 | ||
9 | * of the License, or (at your option) any later version. | ||
10 | * | ||
11 | * This program is distributed in the hope that it will be useful, | ||
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | * GNU General Public License for more details. | ||
15 | * | ||
16 | * You should have received a copy of the GNU General Public License | ||
17 | * along with this program; if not, see <http://www.gnu.org/licenses/>. | ||
18 | */ | ||
19 | |||
20 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | ||
21 | |||
22 | #include <linux/cpu.h> | ||
23 | #include <linux/stacktrace.h> | ||
24 | #include "patch.h" | ||
25 | #include "transition.h" | ||
26 | #include "../sched/sched.h" | ||
27 | |||
28 | #define MAX_STACK_ENTRIES 100 | ||
29 | #define STACK_ERR_BUF_SIZE 128 | ||
30 | |||
31 | extern struct mutex klp_mutex; | ||
32 | |||
33 | struct klp_patch *klp_transition_patch; | ||
34 | |||
35 | static int klp_target_state = KLP_UNDEFINED; | ||
36 | |||
37 | /* | ||
38 | * This work can be performed periodically to finish patching or unpatching any | ||
39 | * "straggler" tasks which failed to transition in the first attempt. | ||
40 | */ | ||
41 | static void klp_transition_work_fn(struct work_struct *work) | ||
42 | { | ||
43 | mutex_lock(&klp_mutex); | ||
44 | |||
45 | if (klp_transition_patch) | ||
46 | klp_try_complete_transition(); | ||
47 | |||
48 | mutex_unlock(&klp_mutex); | ||
49 | } | ||
50 | static DECLARE_DELAYED_WORK(klp_transition_work, klp_transition_work_fn); | ||
51 | |||
52 | /* | ||
53 | * The transition to the target patch state is complete. Clean up the data | ||
54 | * structures. | ||
55 | */ | ||
56 | static void klp_complete_transition(void) | ||
57 | { | ||
58 | struct klp_object *obj; | ||
59 | struct klp_func *func; | ||
60 | struct task_struct *g, *task; | ||
61 | unsigned int cpu; | ||
62 | |||
63 | if (klp_target_state == KLP_UNPATCHED) { | ||
64 | /* | ||
65 | * All tasks have transitioned to KLP_UNPATCHED so we can now | ||
66 | * remove the new functions from the func_stack. | ||
67 | */ | ||
68 | klp_unpatch_objects(klp_transition_patch); | ||
69 | |||
70 | /* | ||
71 | * Make sure klp_ftrace_handler() can no longer see functions | ||
72 | * from this patch on the ops->func_stack. Otherwise, after | ||
73 | * func->transition gets cleared, the handler may choose a | ||
74 | * removed function. | ||
75 | */ | ||
76 | synchronize_rcu(); | ||
77 | } | ||
78 | |||
79 | if (klp_transition_patch->immediate) | ||
80 | goto done; | ||
81 | |||
82 | klp_for_each_object(klp_transition_patch, obj) | ||
83 | klp_for_each_func(obj, func) | ||
84 | func->transition = false; | ||
85 | |||
86 | /* Prevent klp_ftrace_handler() from seeing KLP_UNDEFINED state */ | ||
87 | if (klp_target_state == KLP_PATCHED) | ||
88 | synchronize_rcu(); | ||
89 | |||
90 | read_lock(&tasklist_lock); | ||
91 | for_each_process_thread(g, task) { | ||
92 | WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING)); | ||
93 | task->patch_state = KLP_UNDEFINED; | ||
94 | } | ||
95 | read_unlock(&tasklist_lock); | ||
96 | |||
97 | for_each_possible_cpu(cpu) { | ||
98 | task = idle_task(cpu); | ||
99 | WARN_ON_ONCE(test_tsk_thread_flag(task, TIF_PATCH_PENDING)); | ||
100 | task->patch_state = KLP_UNDEFINED; | ||
101 | } | ||
102 | |||
103 | done: | ||
104 | klp_target_state = KLP_UNDEFINED; | ||
105 | klp_transition_patch = NULL; | ||
106 | } | ||
107 | |||
108 | /* | ||
109 | * This is called in the error path, to cancel a transition before it has | ||
110 | * started, i.e. klp_init_transition() has been called but | ||
111 | * klp_start_transition() hasn't. If the transition *has* been started, | ||
112 | * klp_reverse_transition() should be used instead. | ||
113 | */ | ||
114 | void klp_cancel_transition(void) | ||
115 | { | ||
116 | klp_target_state = !klp_target_state; | ||
117 | klp_complete_transition(); | ||
118 | } | ||
119 | |||
120 | /* | ||
121 | * Switch the patched state of the task to the set of functions in the target | ||
122 | * patch state. | ||
123 | * | ||
124 | * NOTE: If task is not 'current', the caller must ensure the task is inactive. | ||
125 | * Otherwise klp_ftrace_handler() might read the wrong 'patch_state' value. | ||
126 | */ | ||
127 | void klp_update_patch_state(struct task_struct *task) | ||
128 | { | ||
129 | rcu_read_lock(); | ||
130 | |||
131 | /* | ||
132 | * This test_and_clear_tsk_thread_flag() call also serves as a read | ||
133 | * barrier (smp_rmb) for two cases: | ||
134 | * | ||
135 | * 1) Enforce the order of the TIF_PATCH_PENDING read and the | ||
136 | * klp_target_state read. The corresponding write barrier is in | ||
137 | * klp_init_transition(). | ||
138 | * | ||
139 | * 2) Enforce the order of the TIF_PATCH_PENDING read and a future read | ||
140 | * of func->transition, if klp_ftrace_handler() is called later on | ||
141 | * the same CPU. See __klp_disable_patch(). | ||
142 | */ | ||
143 | if (test_and_clear_tsk_thread_flag(task, TIF_PATCH_PENDING)) | ||
144 | task->patch_state = READ_ONCE(klp_target_state); | ||
145 | |||
146 | rcu_read_unlock(); | ||
147 | } | ||
148 | |||
149 | /* | ||
150 | * Determine whether the given stack trace includes any references to a | ||
151 | * to-be-patched or to-be-unpatched function. | ||
152 | */ | ||
153 | static int klp_check_stack_func(struct klp_func *func, | ||
154 | struct stack_trace *trace) | ||
155 | { | ||
156 | unsigned long func_addr, func_size, address; | ||
157 | struct klp_ops *ops; | ||
158 | int i; | ||
159 | |||
160 | if (func->immediate) | ||
161 | return 0; | ||
162 | |||
163 | for (i = 0; i < trace->nr_entries; i++) { | ||
164 | address = trace->entries[i]; | ||
165 | |||
166 | if (klp_target_state == KLP_UNPATCHED) { | ||
167 | /* | ||
168 | * Check for the to-be-unpatched function | ||
169 | * (the func itself). | ||
170 | */ | ||
171 | func_addr = (unsigned long)func->new_func; | ||
172 | func_size = func->new_size; | ||
173 | } else { | ||
174 | /* | ||
175 | * Check for the to-be-patched function | ||
176 | * (the previous func). | ||
177 | */ | ||
178 | ops = klp_find_ops(func->old_addr); | ||
179 | |||
180 | if (list_is_singular(&ops->func_stack)) { | ||
181 | /* original function */ | ||
182 | func_addr = func->old_addr; | ||
183 | func_size = func->old_size; | ||
184 | } else { | ||
185 | /* previously patched function */ | ||
186 | struct klp_func *prev; | ||
187 | |||
188 | prev = list_next_entry(func, stack_node); | ||
189 | func_addr = (unsigned long)prev->new_func; | ||
190 | func_size = prev->new_size; | ||
191 | } | ||
192 | } | ||
193 | |||
194 | if (address >= func_addr && address < func_addr + func_size) | ||
195 | return -EAGAIN; | ||
196 | } | ||
197 | |||
198 | return 0; | ||
199 | } | ||
200 | |||
201 | /* | ||
202 | * Determine whether it's safe to transition the task to the target patch state | ||
203 | * by looking for any to-be-patched or to-be-unpatched functions on its stack. | ||
204 | */ | ||
205 | static int klp_check_stack(struct task_struct *task, char *err_buf) | ||
206 | { | ||
207 | static unsigned long entries[MAX_STACK_ENTRIES]; | ||
208 | struct stack_trace trace; | ||
209 | struct klp_object *obj; | ||
210 | struct klp_func *func; | ||
211 | int ret; | ||
212 | |||
213 | trace.skip = 0; | ||
214 | trace.nr_entries = 0; | ||
215 | trace.max_entries = MAX_STACK_ENTRIES; | ||
216 | trace.entries = entries; | ||
217 | ret = save_stack_trace_tsk_reliable(task, &trace); | ||
218 | WARN_ON_ONCE(ret == -ENOSYS); | ||
219 | if (ret) { | ||
220 | snprintf(err_buf, STACK_ERR_BUF_SIZE, | ||
221 | "%s: %s:%d has an unreliable stack\n", | ||
222 | __func__, task->comm, task->pid); | ||
223 | return ret; | ||
224 | } | ||
225 | |||
226 | klp_for_each_object(klp_transition_patch, obj) { | ||
227 | if (!obj->patched) | ||
228 | continue; | ||
229 | klp_for_each_func(obj, func) { | ||
230 | ret = klp_check_stack_func(func, &trace); | ||
231 | if (ret) { | ||
232 | snprintf(err_buf, STACK_ERR_BUF_SIZE, | ||
233 | "%s: %s:%d is sleeping on function %s\n", | ||
234 | __func__, task->comm, task->pid, | ||
235 | func->old_name); | ||
236 | return ret; | ||
237 | } | ||
238 | } | ||
239 | } | ||
240 | |||
241 | return 0; | ||
242 | } | ||
243 | |||
244 | /* | ||
245 | * Try to safely switch a task to the target patch state. If it's currently | ||
246 | * running, or it's sleeping on a to-be-patched or to-be-unpatched function, or | ||
247 | * if the stack is unreliable, return false. | ||
248 | */ | ||
249 | static bool klp_try_switch_task(struct task_struct *task) | ||
250 | { | ||
251 | struct rq *rq; | ||
252 | struct rq_flags flags; | ||
253 | int ret; | ||
254 | bool success = false; | ||
255 | char err_buf[STACK_ERR_BUF_SIZE]; | ||
256 | |||
257 | err_buf[0] = '\0'; | ||
258 | |||
259 | /* check if this task has already switched over */ | ||
260 | if (task->patch_state == klp_target_state) | ||
261 | return true; | ||
262 | |||
263 | /* | ||
264 | * For arches which don't have reliable stack traces, we have to rely | ||
265 | * on other methods (e.g., switching tasks at kernel exit). | ||
266 | */ | ||
267 | if (!klp_have_reliable_stack()) | ||
268 | return false; | ||
269 | |||
270 | /* | ||
271 | * Now try to check the stack for any to-be-patched or to-be-unpatched | ||
272 | * functions. If all goes well, switch the task to the target patch | ||
273 | * state. | ||
274 | */ | ||
275 | rq = task_rq_lock(task, &flags); | ||
276 | |||
277 | if (task_running(rq, task) && task != current) { | ||
278 | snprintf(err_buf, STACK_ERR_BUF_SIZE, | ||
279 | "%s: %s:%d is running\n", __func__, task->comm, | ||
280 | task->pid); | ||
281 | goto done; | ||
282 | } | ||
283 | |||
284 | ret = klp_check_stack(task, err_buf); | ||
285 | if (ret) | ||
286 | goto done; | ||
287 | |||
288 | success = true; | ||
289 | |||
290 | clear_tsk_thread_flag(task, TIF_PATCH_PENDING); | ||
291 | task->patch_state = klp_target_state; | ||
292 | |||
293 | done: | ||
294 | task_rq_unlock(rq, task, &flags); | ||
295 | |||
296 | /* | ||
297 | * Due to console deadlock issues, pr_debug() can't be used while | ||
298 | * holding the task rq lock. Instead we have to use a temporary buffer | ||
299 | * and print the debug message after releasing the lock. | ||
300 | */ | ||
301 | if (err_buf[0] != '\0') | ||
302 | pr_debug("%s", err_buf); | ||
303 | |||
304 | return success; | ||
305 | |||
306 | } | ||
307 | |||
308 | /* | ||
309 | * Try to switch all remaining tasks to the target patch state by walking the | ||
310 | * stacks of sleeping tasks and looking for any to-be-patched or | ||
311 | * to-be-unpatched functions. If such functions are found, the task can't be | ||
312 | * switched yet. | ||
313 | * | ||
314 | * If any tasks are still stuck in the initial patch state, schedule a retry. | ||
315 | */ | ||
316 | void klp_try_complete_transition(void) | ||
317 | { | ||
318 | unsigned int cpu; | ||
319 | struct task_struct *g, *task; | ||
320 | bool complete = true; | ||
321 | |||
322 | WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED); | ||
323 | |||
324 | /* | ||
325 | * If the patch can be applied or reverted immediately, skip the | ||
326 | * per-task transitions. | ||
327 | */ | ||
328 | if (klp_transition_patch->immediate) | ||
329 | goto success; | ||
330 | |||
331 | /* | ||
332 | * Try to switch the tasks to the target patch state by walking their | ||
333 | * stacks and looking for any to-be-patched or to-be-unpatched | ||
334 | * functions. If such functions are found on a stack, or if the stack | ||
335 | * is deemed unreliable, the task can't be switched yet. | ||
336 | * | ||
337 | * Usually this will transition most (or all) of the tasks on a system | ||
338 | * unless the patch includes changes to a very common function. | ||
339 | */ | ||
340 | read_lock(&tasklist_lock); | ||
341 | for_each_process_thread(g, task) | ||
342 | if (!klp_try_switch_task(task)) | ||
343 | complete = false; | ||
344 | read_unlock(&tasklist_lock); | ||
345 | |||
346 | /* | ||
347 | * Ditto for the idle "swapper" tasks. | ||
348 | */ | ||
349 | get_online_cpus(); | ||
350 | for_each_possible_cpu(cpu) { | ||
351 | task = idle_task(cpu); | ||
352 | if (cpu_online(cpu)) { | ||
353 | if (!klp_try_switch_task(task)) | ||
354 | complete = false; | ||
355 | } else if (task->patch_state != klp_target_state) { | ||
356 | /* offline idle tasks can be switched immediately */ | ||
357 | clear_tsk_thread_flag(task, TIF_PATCH_PENDING); | ||
358 | task->patch_state = klp_target_state; | ||
359 | } | ||
360 | } | ||
361 | put_online_cpus(); | ||
362 | |||
363 | if (!complete) { | ||
364 | /* | ||
365 | * Some tasks weren't able to be switched over. Try again | ||
366 | * later and/or wait for other methods like kernel exit | ||
367 | * switching. | ||
368 | */ | ||
369 | schedule_delayed_work(&klp_transition_work, | ||
370 | round_jiffies_relative(HZ)); | ||
371 | return; | ||
372 | } | ||
373 | |||
374 | success: | ||
375 | pr_notice("'%s': %s complete\n", klp_transition_patch->mod->name, | ||
376 | klp_target_state == KLP_PATCHED ? "patching" : "unpatching"); | ||
377 | |||
378 | /* we're done, now cleanup the data structures */ | ||
379 | klp_complete_transition(); | ||
380 | } | ||
381 | |||
382 | /* | ||
383 | * Start the transition to the specified target patch state so tasks can begin | ||
384 | * switching to it. | ||
385 | */ | ||
386 | void klp_start_transition(void) | ||
387 | { | ||
388 | struct task_struct *g, *task; | ||
389 | unsigned int cpu; | ||
390 | |||
391 | WARN_ON_ONCE(klp_target_state == KLP_UNDEFINED); | ||
392 | |||
393 | pr_notice("'%s': %s...\n", klp_transition_patch->mod->name, | ||
394 | klp_target_state == KLP_PATCHED ? "patching" : "unpatching"); | ||
395 | |||
396 | /* | ||
397 | * If the patch can be applied or reverted immediately, skip the | ||
398 | * per-task transitions. | ||
399 | */ | ||
400 | if (klp_transition_patch->immediate) | ||
401 | return; | ||
402 | |||
403 | /* | ||
404 | * Mark all normal tasks as needing a patch state update. They'll | ||
405 | * switch either in klp_try_complete_transition() or as they exit the | ||
406 | * kernel. | ||
407 | */ | ||
408 | read_lock(&tasklist_lock); | ||
409 | for_each_process_thread(g, task) | ||
410 | if (task->patch_state != klp_target_state) | ||
411 | set_tsk_thread_flag(task, TIF_PATCH_PENDING); | ||
412 | read_unlock(&tasklist_lock); | ||
413 | |||
414 | /* | ||
415 | * Mark all idle tasks as needing a patch state update. They'll switch | ||
416 | * either in klp_try_complete_transition() or at the idle loop switch | ||
417 | * point. | ||
418 | */ | ||
419 | for_each_possible_cpu(cpu) { | ||
420 | task = idle_task(cpu); | ||
421 | if (task->patch_state != klp_target_state) | ||
422 | set_tsk_thread_flag(task, TIF_PATCH_PENDING); | ||
423 | } | ||
424 | } | ||
425 | |||
426 | /* | ||
427 | * Initialize the global target patch state and all tasks to the initial patch | ||
428 | * state, and initialize all function transition states to true in preparation | ||
429 | * for patching or unpatching. | ||
430 | */ | ||
431 | void klp_init_transition(struct klp_patch *patch, int state) | ||
432 | { | ||
433 | struct task_struct *g, *task; | ||
434 | unsigned int cpu; | ||
435 | struct klp_object *obj; | ||
436 | struct klp_func *func; | ||
437 | int initial_state = !state; | ||
438 | |||
439 | WARN_ON_ONCE(klp_target_state != KLP_UNDEFINED); | ||
440 | |||
441 | klp_transition_patch = patch; | ||
442 | |||
443 | /* | ||
444 | * Set the global target patch state which tasks will switch to. This | ||
445 | * has no effect until the TIF_PATCH_PENDING flags get set later. | ||
446 | */ | ||
447 | klp_target_state = state; | ||
448 | |||
449 | /* | ||
450 | * If the patch can be applied or reverted immediately, skip the | ||
451 | * per-task transitions. | ||
452 | */ | ||
453 | if (patch->immediate) | ||
454 | return; | ||
455 | |||
456 | /* | ||
457 | * Initialize all tasks to the initial patch state to prepare them for | ||
458 | * switching to the target state. | ||
459 | */ | ||
460 | read_lock(&tasklist_lock); | ||
461 | for_each_process_thread(g, task) { | ||
462 | WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED); | ||
463 | task->patch_state = initial_state; | ||
464 | } | ||
465 | read_unlock(&tasklist_lock); | ||
466 | |||
467 | /* | ||
468 | * Ditto for the idle "swapper" tasks. | ||
469 | */ | ||
470 | for_each_possible_cpu(cpu) { | ||
471 | task = idle_task(cpu); | ||
472 | WARN_ON_ONCE(task->patch_state != KLP_UNDEFINED); | ||
473 | task->patch_state = initial_state; | ||
474 | } | ||
475 | |||
476 | /* | ||
477 | * Enforce the order of the task->patch_state initializations and the | ||
478 | * func->transition updates to ensure that klp_ftrace_handler() doesn't | ||
479 | * see a func in transition with a task->patch_state of KLP_UNDEFINED. | ||
480 | * | ||
481 | * Also enforce the order of the klp_target_state write and future | ||
482 | * TIF_PATCH_PENDING writes to ensure klp_update_patch_state() doesn't | ||
483 | * set a task->patch_state to KLP_UNDEFINED. | ||
484 | */ | ||
485 | smp_wmb(); | ||
486 | |||
487 | /* | ||
488 | * Set the func transition states so klp_ftrace_handler() will know to | ||
489 | * switch to the transition logic. | ||
490 | * | ||
491 | * When patching, the funcs aren't yet in the func_stack and will be | ||
492 | * made visible to the ftrace handler shortly by the calls to | ||
493 | * klp_patch_object(). | ||
494 | * | ||
495 | * When unpatching, the funcs are already in the func_stack and so are | ||
496 | * already visible to the ftrace handler. | ||
497 | */ | ||
498 | klp_for_each_object(patch, obj) | ||
499 | klp_for_each_func(obj, func) | ||
500 | func->transition = true; | ||
501 | } | ||
502 | |||
503 | /* | ||
504 | * This function can be called in the middle of an existing transition to | ||
505 | * reverse the direction of the target patch state. This can be done to | ||
506 | * effectively cancel an existing enable or disable operation if there are any | ||
507 | * tasks which are stuck in the initial patch state. | ||
508 | */ | ||
509 | void klp_reverse_transition(void) | ||
510 | { | ||
511 | unsigned int cpu; | ||
512 | struct task_struct *g, *task; | ||
513 | |||
514 | klp_transition_patch->enabled = !klp_transition_patch->enabled; | ||
515 | |||
516 | klp_target_state = !klp_target_state; | ||
517 | |||
518 | /* | ||
519 | * Clear all TIF_PATCH_PENDING flags to prevent races caused by | ||
520 | * klp_update_patch_state() running in parallel with | ||
521 | * klp_start_transition(). | ||
522 | */ | ||
523 | read_lock(&tasklist_lock); | ||
524 | for_each_process_thread(g, task) | ||
525 | clear_tsk_thread_flag(task, TIF_PATCH_PENDING); | ||
526 | read_unlock(&tasklist_lock); | ||
527 | |||
528 | for_each_possible_cpu(cpu) | ||
529 | clear_tsk_thread_flag(idle_task(cpu), TIF_PATCH_PENDING); | ||
530 | |||
531 | /* Let any remaining calls to klp_update_patch_state() complete */ | ||
532 | synchronize_rcu(); | ||
533 | |||
534 | klp_start_transition(); | ||
535 | } | ||
536 | |||
537 | /* Called from copy_process() during fork */ | ||
538 | void klp_copy_process(struct task_struct *child) | ||
539 | { | ||
540 | child->patch_state = current->patch_state; | ||
541 | |||
542 | /* TIF_PATCH_PENDING gets copied in setup_thread_stack() */ | ||
543 | } | ||
diff --git a/kernel/livepatch/transition.h b/kernel/livepatch/transition.h new file mode 100644 index 000000000000..ce09b326546c --- /dev/null +++ b/kernel/livepatch/transition.h | |||
@@ -0,0 +1,14 @@ | |||
1 | #ifndef _LIVEPATCH_TRANSITION_H | ||
2 | #define _LIVEPATCH_TRANSITION_H | ||
3 | |||
4 | #include <linux/livepatch.h> | ||
5 | |||
6 | extern struct klp_patch *klp_transition_patch; | ||
7 | |||
8 | void klp_init_transition(struct klp_patch *patch, int state); | ||
9 | void klp_cancel_transition(void); | ||
10 | void klp_start_transition(void); | ||
11 | void klp_try_complete_transition(void); | ||
12 | void klp_reverse_transition(void); | ||
13 | |||
14 | #endif /* _LIVEPATCH_TRANSITION_H */ | ||
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index ac6d5176463d..2a25a9ec2c6e 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c | |||
@@ -10,6 +10,7 @@ | |||
10 | #include <linux/mm.h> | 10 | #include <linux/mm.h> |
11 | #include <linux/stackprotector.h> | 11 | #include <linux/stackprotector.h> |
12 | #include <linux/suspend.h> | 12 | #include <linux/suspend.h> |
13 | #include <linux/livepatch.h> | ||
13 | 14 | ||
14 | #include <asm/tlb.h> | 15 | #include <asm/tlb.h> |
15 | 16 | ||
@@ -265,6 +266,9 @@ static void do_idle(void) | |||
265 | 266 | ||
266 | sched_ttwu_pending(); | 267 | sched_ttwu_pending(); |
267 | schedule_preempt_disabled(); | 268 | schedule_preempt_disabled(); |
269 | |||
270 | if (unlikely(klp_patch_pending(current))) | ||
271 | klp_update_patch_state(current); | ||
268 | } | 272 | } |
269 | 273 | ||
270 | bool cpu_in_idle(unsigned long pc) | 274 | bool cpu_in_idle(unsigned long pc) |
diff --git a/samples/livepatch/livepatch-sample.c b/samples/livepatch/livepatch-sample.c index e34f871e69b1..629e0dca0887 100644 --- a/samples/livepatch/livepatch-sample.c +++ b/samples/livepatch/livepatch-sample.c | |||
@@ -17,6 +17,8 @@ | |||
17 | * along with this program; if not, see <http://www.gnu.org/licenses/>. | 17 | * along with this program; if not, see <http://www.gnu.org/licenses/>. |
18 | */ | 18 | */ |
19 | 19 | ||
20 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | ||
21 | |||
20 | #include <linux/module.h> | 22 | #include <linux/module.h> |
21 | #include <linux/kernel.h> | 23 | #include <linux/kernel.h> |
22 | #include <linux/livepatch.h> | 24 | #include <linux/livepatch.h> |
@@ -69,6 +71,21 @@ static int livepatch_init(void) | |||
69 | { | 71 | { |
70 | int ret; | 72 | int ret; |
71 | 73 | ||
74 | if (!klp_have_reliable_stack() && !patch.immediate) { | ||
75 | /* | ||
76 | * WARNING: Be very careful when using 'patch.immediate' in | ||
77 | * your patches. It's ok to use it for simple patches like | ||
78 | * this, but for more complex patches which change function | ||
79 | * semantics, locking semantics, or data structures, it may not | ||
80 | * be safe. Use of this option will also prevent removal of | ||
81 | * the patch. | ||
82 | * | ||
83 | * See Documentation/livepatch/livepatch.txt for more details. | ||
84 | */ | ||
85 | patch.immediate = true; | ||
86 | pr_notice("The consistency model isn't supported for your architecture. Bypassing safety mechanisms and applying the patch immediately.\n"); | ||
87 | } | ||
88 | |||
72 | ret = klp_register_patch(&patch); | 89 | ret = klp_register_patch(&patch); |
73 | if (ret) | 90 | if (ret) |
74 | return ret; | 91 | return ret; |