aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/power
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/power')
-rw-r--r--kernel/power/Kconfig8
-rw-r--r--kernel/power/Makefile4
-rw-r--r--kernel/power/console.c4
-rw-r--r--kernel/power/hibernate.c54
-rw-r--r--kernel/power/main.c103
-rw-r--r--kernel/power/power.h4
-rw-r--r--kernel/power/process.c30
-rw-r--r--kernel/power/qos.c491
-rw-r--r--kernel/power/snapshot.c18
-rw-r--r--kernel/power/suspend.c19
-rw-r--r--kernel/power/swap.c818
-rw-r--r--kernel/power/user.c1
12 files changed, 1349 insertions, 205 deletions
diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig
index 3744c594b19b..deb5461e3216 100644
--- a/kernel/power/Kconfig
+++ b/kernel/power/Kconfig
@@ -27,6 +27,7 @@ config HIBERNATION
27 select HIBERNATE_CALLBACKS 27 select HIBERNATE_CALLBACKS
28 select LZO_COMPRESS 28 select LZO_COMPRESS
29 select LZO_DECOMPRESS 29 select LZO_DECOMPRESS
30 select CRC32
30 ---help--- 31 ---help---
31 Enable the suspend to disk (STD) functionality, which is usually 32 Enable the suspend to disk (STD) functionality, which is usually
32 called "hibernation" in user interfaces. STD checkpoints the 33 called "hibernation" in user interfaces. STD checkpoints the
@@ -65,6 +66,9 @@ config HIBERNATION
65 66
66 For more information take a look at <file:Documentation/power/swsusp.txt>. 67 For more information take a look at <file:Documentation/power/swsusp.txt>.
67 68
69config ARCH_SAVE_PAGE_KEYS
70 bool
71
68config PM_STD_PARTITION 72config PM_STD_PARTITION
69 string "Default resume partition" 73 string "Default resume partition"
70 depends on HIBERNATION 74 depends on HIBERNATION
@@ -235,3 +239,7 @@ config PM_GENERIC_DOMAINS
235config PM_GENERIC_DOMAINS_RUNTIME 239config PM_GENERIC_DOMAINS_RUNTIME
236 def_bool y 240 def_bool y
237 depends on PM_RUNTIME && PM_GENERIC_DOMAINS 241 depends on PM_RUNTIME && PM_GENERIC_DOMAINS
242
243config CPU_PM
244 bool
245 depends on SUSPEND || CPU_IDLE
diff --git a/kernel/power/Makefile b/kernel/power/Makefile
index c5ebc6a90643..07e0e28ffba7 100644
--- a/kernel/power/Makefile
+++ b/kernel/power/Makefile
@@ -1,8 +1,8 @@
1 1
2ccflags-$(CONFIG_PM_DEBUG) := -DDEBUG 2ccflags-$(CONFIG_PM_DEBUG) := -DDEBUG
3 3
4obj-$(CONFIG_PM) += main.o 4obj-$(CONFIG_PM) += main.o qos.o
5obj-$(CONFIG_PM_SLEEP) += console.o 5obj-$(CONFIG_VT_CONSOLE_SLEEP) += console.o
6obj-$(CONFIG_FREEZER) += process.o 6obj-$(CONFIG_FREEZER) += process.o
7obj-$(CONFIG_SUSPEND) += suspend.o 7obj-$(CONFIG_SUSPEND) += suspend.o
8obj-$(CONFIG_PM_TEST_SUSPEND) += suspend_test.o 8obj-$(CONFIG_PM_TEST_SUSPEND) += suspend_test.o
diff --git a/kernel/power/console.c b/kernel/power/console.c
index 218e5af90156..b1dc456474b5 100644
--- a/kernel/power/console.c
+++ b/kernel/power/console.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * drivers/power/process.c - Functions for saving/restoring console. 2 * Functions for saving/restoring console.
3 * 3 *
4 * Originally from swsusp. 4 * Originally from swsusp.
5 */ 5 */
@@ -10,7 +10,6 @@
10#include <linux/module.h> 10#include <linux/module.h>
11#include "power.h" 11#include "power.h"
12 12
13#if defined(CONFIG_VT) && defined(CONFIG_VT_CONSOLE)
14#define SUSPEND_CONSOLE (MAX_NR_CONSOLES-1) 13#define SUSPEND_CONSOLE (MAX_NR_CONSOLES-1)
15 14
16static int orig_fgconsole, orig_kmsg; 15static int orig_fgconsole, orig_kmsg;
@@ -32,4 +31,3 @@ void pm_restore_console(void)
32 vt_kmsg_redirect(orig_kmsg); 31 vt_kmsg_redirect(orig_kmsg);
33 } 32 }
34} 33}
35#endif
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index 8f7b1db1ece1..b4511b6d3ef9 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -9,11 +9,13 @@
9 * This file is released under the GPLv2. 9 * This file is released under the GPLv2.
10 */ 10 */
11 11
12#include <linux/export.h>
12#include <linux/suspend.h> 13#include <linux/suspend.h>
13#include <linux/syscalls.h> 14#include <linux/syscalls.h>
14#include <linux/reboot.h> 15#include <linux/reboot.h>
15#include <linux/string.h> 16#include <linux/string.h>
16#include <linux/device.h> 17#include <linux/device.h>
18#include <linux/async.h>
17#include <linux/kmod.h> 19#include <linux/kmod.h>
18#include <linux/delay.h> 20#include <linux/delay.h>
19#include <linux/fs.h> 21#include <linux/fs.h>
@@ -29,12 +31,14 @@
29#include "power.h" 31#include "power.h"
30 32
31 33
32static int nocompress = 0; 34static int nocompress;
33static int noresume = 0; 35static int noresume;
36static int resume_wait;
37static int resume_delay;
34static char resume_file[256] = CONFIG_PM_STD_PARTITION; 38static char resume_file[256] = CONFIG_PM_STD_PARTITION;
35dev_t swsusp_resume_device; 39dev_t swsusp_resume_device;
36sector_t swsusp_resume_block; 40sector_t swsusp_resume_block;
37int in_suspend __nosavedata = 0; 41int in_suspend __nosavedata;
38 42
39enum { 43enum {
40 HIBERNATION_INVALID, 44 HIBERNATION_INVALID,
@@ -334,13 +338,17 @@ int hibernation_snapshot(int platform_mode)
334 if (error) 338 if (error)
335 goto Close; 339 goto Close;
336 340
337 error = dpm_prepare(PMSG_FREEZE);
338 if (error)
339 goto Complete_devices;
340
341 /* Preallocate image memory before shutting down devices. */ 341 /* Preallocate image memory before shutting down devices. */
342 error = hibernate_preallocate_memory(); 342 error = hibernate_preallocate_memory();
343 if (error) 343 if (error)
344 goto Close;
345
346 error = freeze_kernel_threads();
347 if (error)
348 goto Close;
349
350 error = dpm_prepare(PMSG_FREEZE);
351 if (error)
344 goto Complete_devices; 352 goto Complete_devices;
345 353
346 suspend_console(); 354 suspend_console();
@@ -463,7 +471,7 @@ static int resume_target_kernel(bool platform_mode)
463 * @platform_mode: If set, use platform driver to prepare for the transition. 471 * @platform_mode: If set, use platform driver to prepare for the transition.
464 * 472 *
465 * This routine must be called with pm_mutex held. If it is successful, control 473 * This routine must be called with pm_mutex held. If it is successful, control
466 * reappears in the restored target kernel in hibernation_snaphot(). 474 * reappears in the restored target kernel in hibernation_snapshot().
467 */ 475 */
468int hibernation_restore(int platform_mode) 476int hibernation_restore(int platform_mode)
469{ 477{
@@ -650,6 +658,9 @@ int hibernate(void)
650 flags |= SF_PLATFORM_MODE; 658 flags |= SF_PLATFORM_MODE;
651 if (nocompress) 659 if (nocompress)
652 flags |= SF_NOCOMPRESS_MODE; 660 flags |= SF_NOCOMPRESS_MODE;
661 else
662 flags |= SF_CRC32_MODE;
663
653 pr_debug("PM: writing image.\n"); 664 pr_debug("PM: writing image.\n");
654 error = swsusp_write(flags); 665 error = swsusp_write(flags);
655 swsusp_free(); 666 swsusp_free();
@@ -724,6 +735,12 @@ static int software_resume(void)
724 735
725 pr_debug("PM: Checking hibernation image partition %s\n", resume_file); 736 pr_debug("PM: Checking hibernation image partition %s\n", resume_file);
726 737
738 if (resume_delay) {
739 printk(KERN_INFO "Waiting %dsec before reading resume device...\n",
740 resume_delay);
741 ssleep(resume_delay);
742 }
743
727 /* Check if the device is there */ 744 /* Check if the device is there */
728 swsusp_resume_device = name_to_dev_t(resume_file); 745 swsusp_resume_device = name_to_dev_t(resume_file);
729 if (!swsusp_resume_device) { 746 if (!swsusp_resume_device) {
@@ -732,6 +749,13 @@ static int software_resume(void)
732 * to wait for this to finish. 749 * to wait for this to finish.
733 */ 750 */
734 wait_for_device_probe(); 751 wait_for_device_probe();
752
753 if (resume_wait) {
754 while ((swsusp_resume_device = name_to_dev_t(resume_file)) == 0)
755 msleep(10);
756 async_synchronize_full();
757 }
758
735 /* 759 /*
736 * We can't depend on SCSI devices being available after loading 760 * We can't depend on SCSI devices being available after loading
737 * one of their modules until scsi_complete_async_scans() is 761 * one of their modules until scsi_complete_async_scans() is
@@ -1060,7 +1084,21 @@ static int __init noresume_setup(char *str)
1060 return 1; 1084 return 1;
1061} 1085}
1062 1086
1087static int __init resumewait_setup(char *str)
1088{
1089 resume_wait = 1;
1090 return 1;
1091}
1092
1093static int __init resumedelay_setup(char *str)
1094{
1095 resume_delay = simple_strtoul(str, NULL, 0);
1096 return 1;
1097}
1098
1063__setup("noresume", noresume_setup); 1099__setup("noresume", noresume_setup);
1064__setup("resume_offset=", resume_offset_setup); 1100__setup("resume_offset=", resume_offset_setup);
1065__setup("resume=", resume_setup); 1101__setup("resume=", resume_setup);
1066__setup("hibernate=", hibernate_setup); 1102__setup("hibernate=", hibernate_setup);
1103__setup("resumewait", resumewait_setup);
1104__setup("resumedelay=", resumedelay_setup);
diff --git a/kernel/power/main.c b/kernel/power/main.c
index 6c601f871964..71f49fe4377e 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -8,10 +8,13 @@
8 * 8 *
9 */ 9 */
10 10
11#include <linux/export.h>
11#include <linux/kobject.h> 12#include <linux/kobject.h>
12#include <linux/string.h> 13#include <linux/string.h>
13#include <linux/resume-trace.h> 14#include <linux/resume-trace.h>
14#include <linux/workqueue.h> 15#include <linux/workqueue.h>
16#include <linux/debugfs.h>
17#include <linux/seq_file.h>
15 18
16#include "power.h" 19#include "power.h"
17 20
@@ -131,6 +134,101 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr,
131power_attr(pm_test); 134power_attr(pm_test);
132#endif /* CONFIG_PM_DEBUG */ 135#endif /* CONFIG_PM_DEBUG */
133 136
137#ifdef CONFIG_DEBUG_FS
138static char *suspend_step_name(enum suspend_stat_step step)
139{
140 switch (step) {
141 case SUSPEND_FREEZE:
142 return "freeze";
143 case SUSPEND_PREPARE:
144 return "prepare";
145 case SUSPEND_SUSPEND:
146 return "suspend";
147 case SUSPEND_SUSPEND_NOIRQ:
148 return "suspend_noirq";
149 case SUSPEND_RESUME_NOIRQ:
150 return "resume_noirq";
151 case SUSPEND_RESUME:
152 return "resume";
153 default:
154 return "";
155 }
156}
157
158static int suspend_stats_show(struct seq_file *s, void *unused)
159{
160 int i, index, last_dev, last_errno, last_step;
161
162 last_dev = suspend_stats.last_failed_dev + REC_FAILED_NUM - 1;
163 last_dev %= REC_FAILED_NUM;
164 last_errno = suspend_stats.last_failed_errno + REC_FAILED_NUM - 1;
165 last_errno %= REC_FAILED_NUM;
166 last_step = suspend_stats.last_failed_step + REC_FAILED_NUM - 1;
167 last_step %= REC_FAILED_NUM;
168 seq_printf(s, "%s: %d\n%s: %d\n%s: %d\n%s: %d\n"
169 "%s: %d\n%s: %d\n%s: %d\n%s: %d\n",
170 "success", suspend_stats.success,
171 "fail", suspend_stats.fail,
172 "failed_freeze", suspend_stats.failed_freeze,
173 "failed_prepare", suspend_stats.failed_prepare,
174 "failed_suspend", suspend_stats.failed_suspend,
175 "failed_suspend_noirq",
176 suspend_stats.failed_suspend_noirq,
177 "failed_resume", suspend_stats.failed_resume,
178 "failed_resume_noirq",
179 suspend_stats.failed_resume_noirq);
180 seq_printf(s, "failures:\n last_failed_dev:\t%-s\n",
181 suspend_stats.failed_devs[last_dev]);
182 for (i = 1; i < REC_FAILED_NUM; i++) {
183 index = last_dev + REC_FAILED_NUM - i;
184 index %= REC_FAILED_NUM;
185 seq_printf(s, "\t\t\t%-s\n",
186 suspend_stats.failed_devs[index]);
187 }
188 seq_printf(s, " last_failed_errno:\t%-d\n",
189 suspend_stats.errno[last_errno]);
190 for (i = 1; i < REC_FAILED_NUM; i++) {
191 index = last_errno + REC_FAILED_NUM - i;
192 index %= REC_FAILED_NUM;
193 seq_printf(s, "\t\t\t%-d\n",
194 suspend_stats.errno[index]);
195 }
196 seq_printf(s, " last_failed_step:\t%-s\n",
197 suspend_step_name(
198 suspend_stats.failed_steps[last_step]));
199 for (i = 1; i < REC_FAILED_NUM; i++) {
200 index = last_step + REC_FAILED_NUM - i;
201 index %= REC_FAILED_NUM;
202 seq_printf(s, "\t\t\t%-s\n",
203 suspend_step_name(
204 suspend_stats.failed_steps[index]));
205 }
206
207 return 0;
208}
209
210static int suspend_stats_open(struct inode *inode, struct file *file)
211{
212 return single_open(file, suspend_stats_show, NULL);
213}
214
215static const struct file_operations suspend_stats_operations = {
216 .open = suspend_stats_open,
217 .read = seq_read,
218 .llseek = seq_lseek,
219 .release = single_release,
220};
221
222static int __init pm_debugfs_init(void)
223{
224 debugfs_create_file("suspend_stats", S_IFREG | S_IRUGO,
225 NULL, NULL, &suspend_stats_operations);
226 return 0;
227}
228
229late_initcall(pm_debugfs_init);
230#endif /* CONFIG_DEBUG_FS */
231
134#endif /* CONFIG_PM_SLEEP */ 232#endif /* CONFIG_PM_SLEEP */
135 233
136struct kobject *power_kobj; 234struct kobject *power_kobj;
@@ -194,6 +292,11 @@ static ssize_t state_store(struct kobject *kobj, struct kobj_attribute *attr,
194 } 292 }
195 if (state < PM_SUSPEND_MAX && *s) 293 if (state < PM_SUSPEND_MAX && *s)
196 error = enter_state(state); 294 error = enter_state(state);
295 if (error) {
296 suspend_stats.fail++;
297 dpm_save_failed_errno(error);
298 } else
299 suspend_stats.success++;
197#endif 300#endif
198 301
199 Exit: 302 Exit:
diff --git a/kernel/power/power.h b/kernel/power/power.h
index 9a00a0a26280..23a2db1ec442 100644
--- a/kernel/power/power.h
+++ b/kernel/power/power.h
@@ -146,6 +146,7 @@ extern int swsusp_swap_in_use(void);
146 */ 146 */
147#define SF_PLATFORM_MODE 1 147#define SF_PLATFORM_MODE 1
148#define SF_NOCOMPRESS_MODE 2 148#define SF_NOCOMPRESS_MODE 2
149#define SF_CRC32_MODE 4
149 150
150/* kernel/power/hibernate.c */ 151/* kernel/power/hibernate.c */
151extern int swsusp_check(void); 152extern int swsusp_check(void);
@@ -228,7 +229,8 @@ extern int pm_test_level;
228#ifdef CONFIG_SUSPEND_FREEZER 229#ifdef CONFIG_SUSPEND_FREEZER
229static inline int suspend_freeze_processes(void) 230static inline int suspend_freeze_processes(void)
230{ 231{
231 return freeze_processes(); 232 int error = freeze_processes();
233 return error ? : freeze_kernel_threads();
232} 234}
233 235
234static inline void suspend_thaw_processes(void) 236static inline void suspend_thaw_processes(void)
diff --git a/kernel/power/process.c b/kernel/power/process.c
index 0cf3a27a6c9d..addbbe5531bc 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -135,7 +135,7 @@ static int try_to_freeze_tasks(bool sig_only)
135} 135}
136 136
137/** 137/**
138 * freeze_processes - tell processes to enter the refrigerator 138 * freeze_processes - Signal user space processes to enter the refrigerator.
139 */ 139 */
140int freeze_processes(void) 140int freeze_processes(void)
141{ 141{
@@ -143,20 +143,30 @@ int freeze_processes(void)
143 143
144 printk("Freezing user space processes ... "); 144 printk("Freezing user space processes ... ");
145 error = try_to_freeze_tasks(true); 145 error = try_to_freeze_tasks(true);
146 if (error) 146 if (!error) {
147 goto Exit; 147 printk("done.");
148 printk("done.\n"); 148 oom_killer_disable();
149 }
150 printk("\n");
151 BUG_ON(in_atomic());
152
153 return error;
154}
155
156/**
157 * freeze_kernel_threads - Make freezable kernel threads go to the refrigerator.
158 */
159int freeze_kernel_threads(void)
160{
161 int error;
149 162
150 printk("Freezing remaining freezable tasks ... "); 163 printk("Freezing remaining freezable tasks ... ");
151 error = try_to_freeze_tasks(false); 164 error = try_to_freeze_tasks(false);
152 if (error) 165 if (!error)
153 goto Exit; 166 printk("done.");
154 printk("done.");
155 167
156 oom_killer_disable();
157 Exit:
158 BUG_ON(in_atomic());
159 printk("\n"); 168 printk("\n");
169 BUG_ON(in_atomic());
160 170
161 return error; 171 return error;
162} 172}
diff --git a/kernel/power/qos.c b/kernel/power/qos.c
new file mode 100644
index 000000000000..995e3bd3417b
--- /dev/null
+++ b/kernel/power/qos.c
@@ -0,0 +1,491 @@
1/*
2 * This module exposes the interface to kernel space for specifying
3 * QoS dependencies. It provides infrastructure for registration of:
4 *
5 * Dependents on a QoS value : register requests
6 * Watchers of QoS value : get notified when target QoS value changes
7 *
8 * This QoS design is best effort based. Dependents register their QoS needs.
9 * Watchers register to keep track of the current QoS needs of the system.
10 *
11 * There are 3 basic classes of QoS parameter: latency, timeout, throughput
12 * each have defined units:
13 * latency: usec
14 * timeout: usec <-- currently not used.
15 * throughput: kbs (kilo byte / sec)
16 *
17 * There are lists of pm_qos_objects each one wrapping requests, notifiers
18 *
19 * User mode requests on a QOS parameter register themselves to the
20 * subsystem by opening the device node /dev/... and writing there request to
21 * the node. As long as the process holds a file handle open to the node the
22 * client continues to be accounted for. Upon file release the usermode
23 * request is removed and a new qos target is computed. This way when the
24 * request that the application has is cleaned up when closes the file
25 * pointer or exits the pm_qos_object will get an opportunity to clean up.
26 *
27 * Mark Gross <mgross@linux.intel.com>
28 */
29
30/*#define DEBUG*/
31
32#include <linux/pm_qos.h>
33#include <linux/sched.h>
34#include <linux/spinlock.h>
35#include <linux/slab.h>
36#include <linux/time.h>
37#include <linux/fs.h>
38#include <linux/device.h>
39#include <linux/miscdevice.h>
40#include <linux/string.h>
41#include <linux/platform_device.h>
42#include <linux/init.h>
43#include <linux/kernel.h>
44
45#include <linux/uaccess.h>
46#include <linux/export.h>
47
48/*
49 * locking rule: all changes to constraints or notifiers lists
50 * or pm_qos_object list and pm_qos_objects need to happen with pm_qos_lock
51 * held, taken with _irqsave. One lock to rule them all
52 */
53struct pm_qos_object {
54 struct pm_qos_constraints *constraints;
55 struct miscdevice pm_qos_power_miscdev;
56 char *name;
57};
58
59static DEFINE_SPINLOCK(pm_qos_lock);
60
61static struct pm_qos_object null_pm_qos;
62
63static BLOCKING_NOTIFIER_HEAD(cpu_dma_lat_notifier);
64static struct pm_qos_constraints cpu_dma_constraints = {
65 .list = PLIST_HEAD_INIT(cpu_dma_constraints.list),
66 .target_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
67 .default_value = PM_QOS_CPU_DMA_LAT_DEFAULT_VALUE,
68 .type = PM_QOS_MIN,
69 .notifiers = &cpu_dma_lat_notifier,
70};
71static struct pm_qos_object cpu_dma_pm_qos = {
72 .constraints = &cpu_dma_constraints,
73 .name = "cpu_dma_latency",
74};
75
76static BLOCKING_NOTIFIER_HEAD(network_lat_notifier);
77static struct pm_qos_constraints network_lat_constraints = {
78 .list = PLIST_HEAD_INIT(network_lat_constraints.list),
79 .target_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
80 .default_value = PM_QOS_NETWORK_LAT_DEFAULT_VALUE,
81 .type = PM_QOS_MIN,
82 .notifiers = &network_lat_notifier,
83};
84static struct pm_qos_object network_lat_pm_qos = {
85 .constraints = &network_lat_constraints,
86 .name = "network_latency",
87};
88
89
90static BLOCKING_NOTIFIER_HEAD(network_throughput_notifier);
91static struct pm_qos_constraints network_tput_constraints = {
92 .list = PLIST_HEAD_INIT(network_tput_constraints.list),
93 .target_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
94 .default_value = PM_QOS_NETWORK_THROUGHPUT_DEFAULT_VALUE,
95 .type = PM_QOS_MAX,
96 .notifiers = &network_throughput_notifier,
97};
98static struct pm_qos_object network_throughput_pm_qos = {
99 .constraints = &network_tput_constraints,
100 .name = "network_throughput",
101};
102
103
104static struct pm_qos_object *pm_qos_array[] = {
105 &null_pm_qos,
106 &cpu_dma_pm_qos,
107 &network_lat_pm_qos,
108 &network_throughput_pm_qos
109};
110
111static ssize_t pm_qos_power_write(struct file *filp, const char __user *buf,
112 size_t count, loff_t *f_pos);
113static ssize_t pm_qos_power_read(struct file *filp, char __user *buf,
114 size_t count, loff_t *f_pos);
115static int pm_qos_power_open(struct inode *inode, struct file *filp);
116static int pm_qos_power_release(struct inode *inode, struct file *filp);
117
118static const struct file_operations pm_qos_power_fops = {
119 .write = pm_qos_power_write,
120 .read = pm_qos_power_read,
121 .open = pm_qos_power_open,
122 .release = pm_qos_power_release,
123 .llseek = noop_llseek,
124};
125
126/* unlocked internal variant */
127static inline int pm_qos_get_value(struct pm_qos_constraints *c)
128{
129 if (plist_head_empty(&c->list))
130 return c->default_value;
131
132 switch (c->type) {
133 case PM_QOS_MIN:
134 return plist_first(&c->list)->prio;
135
136 case PM_QOS_MAX:
137 return plist_last(&c->list)->prio;
138
139 default:
140 /* runtime check for not using enum */
141 BUG();
142 }
143}
144
145s32 pm_qos_read_value(struct pm_qos_constraints *c)
146{
147 return c->target_value;
148}
149
150static inline void pm_qos_set_value(struct pm_qos_constraints *c, s32 value)
151{
152 c->target_value = value;
153}
154
155/**
156 * pm_qos_update_target - manages the constraints list and calls the notifiers
157 * if needed
158 * @c: constraints data struct
159 * @node: request to add to the list, to update or to remove
160 * @action: action to take on the constraints list
161 * @value: value of the request to add or update
162 *
163 * This function returns 1 if the aggregated constraint value has changed, 0
164 * otherwise.
165 */
166int pm_qos_update_target(struct pm_qos_constraints *c, struct plist_node *node,
167 enum pm_qos_req_action action, int value)
168{
169 unsigned long flags;
170 int prev_value, curr_value, new_value;
171
172 spin_lock_irqsave(&pm_qos_lock, flags);
173 prev_value = pm_qos_get_value(c);
174 if (value == PM_QOS_DEFAULT_VALUE)
175 new_value = c->default_value;
176 else
177 new_value = value;
178
179 switch (action) {
180 case PM_QOS_REMOVE_REQ:
181 plist_del(node, &c->list);
182 break;
183 case PM_QOS_UPDATE_REQ:
184 /*
185 * to change the list, we atomically remove, reinit
186 * with new value and add, then see if the extremal
187 * changed
188 */
189 plist_del(node, &c->list);
190 case PM_QOS_ADD_REQ:
191 plist_node_init(node, new_value);
192 plist_add(node, &c->list);
193 break;
194 default:
195 /* no action */
196 ;
197 }
198
199 curr_value = pm_qos_get_value(c);
200 pm_qos_set_value(c, curr_value);
201
202 spin_unlock_irqrestore(&pm_qos_lock, flags);
203
204 if (prev_value != curr_value) {
205 blocking_notifier_call_chain(c->notifiers,
206 (unsigned long)curr_value,
207 NULL);
208 return 1;
209 } else {
210 return 0;
211 }
212}
213
214/**
215 * pm_qos_request - returns current system wide qos expectation
216 * @pm_qos_class: identification of which qos value is requested
217 *
218 * This function returns the current target value.
219 */
220int pm_qos_request(int pm_qos_class)
221{
222 return pm_qos_read_value(pm_qos_array[pm_qos_class]->constraints);
223}
224EXPORT_SYMBOL_GPL(pm_qos_request);
225
226int pm_qos_request_active(struct pm_qos_request *req)
227{
228 return req->pm_qos_class != 0;
229}
230EXPORT_SYMBOL_GPL(pm_qos_request_active);
231
232/**
233 * pm_qos_add_request - inserts new qos request into the list
234 * @req: pointer to a preallocated handle
235 * @pm_qos_class: identifies which list of qos request to use
236 * @value: defines the qos request
237 *
238 * This function inserts a new entry in the pm_qos_class list of requested qos
239 * performance characteristics. It recomputes the aggregate QoS expectations
240 * for the pm_qos_class of parameters and initializes the pm_qos_request
241 * handle. Caller needs to save this handle for later use in updates and
242 * removal.
243 */
244
245void pm_qos_add_request(struct pm_qos_request *req,
246 int pm_qos_class, s32 value)
247{
248 if (!req) /*guard against callers passing in null */
249 return;
250
251 if (pm_qos_request_active(req)) {
252 WARN(1, KERN_ERR "pm_qos_add_request() called for already added request\n");
253 return;
254 }
255 req->pm_qos_class = pm_qos_class;
256 pm_qos_update_target(pm_qos_array[pm_qos_class]->constraints,
257 &req->node, PM_QOS_ADD_REQ, value);
258}
259EXPORT_SYMBOL_GPL(pm_qos_add_request);
260
261/**
262 * pm_qos_update_request - modifies an existing qos request
263 * @req : handle to list element holding a pm_qos request to use
264 * @value: defines the qos request
265 *
266 * Updates an existing qos request for the pm_qos_class of parameters along
267 * with updating the target pm_qos_class value.
268 *
269 * Attempts are made to make this code callable on hot code paths.
270 */
271void pm_qos_update_request(struct pm_qos_request *req,
272 s32 new_value)
273{
274 if (!req) /*guard against callers passing in null */
275 return;
276
277 if (!pm_qos_request_active(req)) {
278 WARN(1, KERN_ERR "pm_qos_update_request() called for unknown object\n");
279 return;
280 }
281
282 if (new_value != req->node.prio)
283 pm_qos_update_target(
284 pm_qos_array[req->pm_qos_class]->constraints,
285 &req->node, PM_QOS_UPDATE_REQ, new_value);
286}
287EXPORT_SYMBOL_GPL(pm_qos_update_request);
288
289/**
290 * pm_qos_remove_request - modifies an existing qos request
291 * @req: handle to request list element
292 *
293 * Will remove pm qos request from the list of constraints and
294 * recompute the current target value for the pm_qos_class. Call this
295 * on slow code paths.
296 */
297void pm_qos_remove_request(struct pm_qos_request *req)
298{
299 if (!req) /*guard against callers passing in null */
300 return;
301 /* silent return to keep pcm code cleaner */
302
303 if (!pm_qos_request_active(req)) {
304 WARN(1, KERN_ERR "pm_qos_remove_request() called for unknown object\n");
305 return;
306 }
307
308 pm_qos_update_target(pm_qos_array[req->pm_qos_class]->constraints,
309 &req->node, PM_QOS_REMOVE_REQ,
310 PM_QOS_DEFAULT_VALUE);
311 memset(req, 0, sizeof(*req));
312}
313EXPORT_SYMBOL_GPL(pm_qos_remove_request);
314
315/**
316 * pm_qos_add_notifier - sets notification entry for changes to target value
317 * @pm_qos_class: identifies which qos target changes should be notified.
318 * @notifier: notifier block managed by caller.
319 *
320 * will register the notifier into a notification chain that gets called
321 * upon changes to the pm_qos_class target value.
322 */
323int pm_qos_add_notifier(int pm_qos_class, struct notifier_block *notifier)
324{
325 int retval;
326
327 retval = blocking_notifier_chain_register(
328 pm_qos_array[pm_qos_class]->constraints->notifiers,
329 notifier);
330
331 return retval;
332}
333EXPORT_SYMBOL_GPL(pm_qos_add_notifier);
334
335/**
336 * pm_qos_remove_notifier - deletes notification entry from chain.
337 * @pm_qos_class: identifies which qos target changes are notified.
338 * @notifier: notifier block to be removed.
339 *
340 * will remove the notifier from the notification chain that gets called
341 * upon changes to the pm_qos_class target value.
342 */
343int pm_qos_remove_notifier(int pm_qos_class, struct notifier_block *notifier)
344{
345 int retval;
346
347 retval = blocking_notifier_chain_unregister(
348 pm_qos_array[pm_qos_class]->constraints->notifiers,
349 notifier);
350
351 return retval;
352}
353EXPORT_SYMBOL_GPL(pm_qos_remove_notifier);
354
355/* User space interface to PM QoS classes via misc devices */
356static int register_pm_qos_misc(struct pm_qos_object *qos)
357{
358 qos->pm_qos_power_miscdev.minor = MISC_DYNAMIC_MINOR;
359 qos->pm_qos_power_miscdev.name = qos->name;
360 qos->pm_qos_power_miscdev.fops = &pm_qos_power_fops;
361
362 return misc_register(&qos->pm_qos_power_miscdev);
363}
364
365static int find_pm_qos_object_by_minor(int minor)
366{
367 int pm_qos_class;
368
369 for (pm_qos_class = 0;
370 pm_qos_class < PM_QOS_NUM_CLASSES; pm_qos_class++) {
371 if (minor ==
372 pm_qos_array[pm_qos_class]->pm_qos_power_miscdev.minor)
373 return pm_qos_class;
374 }
375 return -1;
376}
377
378static int pm_qos_power_open(struct inode *inode, struct file *filp)
379{
380 long pm_qos_class;
381
382 pm_qos_class = find_pm_qos_object_by_minor(iminor(inode));
383 if (pm_qos_class >= 0) {
384 struct pm_qos_request *req = kzalloc(sizeof(*req), GFP_KERNEL);
385 if (!req)
386 return -ENOMEM;
387
388 pm_qos_add_request(req, pm_qos_class, PM_QOS_DEFAULT_VALUE);
389 filp->private_data = req;
390
391 return 0;
392 }
393 return -EPERM;
394}
395
396static int pm_qos_power_release(struct inode *inode, struct file *filp)
397{
398 struct pm_qos_request *req;
399
400 req = filp->private_data;
401 pm_qos_remove_request(req);
402 kfree(req);
403
404 return 0;
405}
406
407
408static ssize_t pm_qos_power_read(struct file *filp, char __user *buf,
409 size_t count, loff_t *f_pos)
410{
411 s32 value;
412 unsigned long flags;
413 struct pm_qos_request *req = filp->private_data;
414
415 if (!req)
416 return -EINVAL;
417 if (!pm_qos_request_active(req))
418 return -EINVAL;
419
420 spin_lock_irqsave(&pm_qos_lock, flags);
421 value = pm_qos_get_value(pm_qos_array[req->pm_qos_class]->constraints);
422 spin_unlock_irqrestore(&pm_qos_lock, flags);
423
424 return simple_read_from_buffer(buf, count, f_pos, &value, sizeof(s32));
425}
426
427static ssize_t pm_qos_power_write(struct file *filp, const char __user *buf,
428 size_t count, loff_t *f_pos)
429{
430 s32 value;
431 struct pm_qos_request *req;
432
433 if (count == sizeof(s32)) {
434 if (copy_from_user(&value, buf, sizeof(s32)))
435 return -EFAULT;
436 } else if (count <= 11) { /* ASCII perhaps? */
437 char ascii_value[11];
438 unsigned long int ulval;
439 int ret;
440
441 if (copy_from_user(ascii_value, buf, count))
442 return -EFAULT;
443
444 if (count > 10) {
445 if (ascii_value[10] == '\n')
446 ascii_value[10] = '\0';
447 else
448 return -EINVAL;
449 } else {
450 ascii_value[count] = '\0';
451 }
452 ret = strict_strtoul(ascii_value, 16, &ulval);
453 if (ret) {
454 pr_debug("%s, 0x%lx, 0x%x\n", ascii_value, ulval, ret);
455 return -EINVAL;
456 }
457 value = (s32)lower_32_bits(ulval);
458 } else {
459 return -EINVAL;
460 }
461
462 req = filp->private_data;
463 pm_qos_update_request(req, value);
464
465 return count;
466}
467
468
469static int __init pm_qos_power_init(void)
470{
471 int ret = 0;
472
473 ret = register_pm_qos_misc(&cpu_dma_pm_qos);
474 if (ret < 0) {
475 printk(KERN_ERR "pm_qos_param: cpu_dma_latency setup failed\n");
476 return ret;
477 }
478 ret = register_pm_qos_misc(&network_lat_pm_qos);
479 if (ret < 0) {
480 printk(KERN_ERR "pm_qos_param: network_latency setup failed\n");
481 return ret;
482 }
483 ret = register_pm_qos_misc(&network_throughput_pm_qos);
484 if (ret < 0)
485 printk(KERN_ERR
486 "pm_qos_param: network_throughput setup failed\n");
487
488 return ret;
489}
490
491late_initcall(pm_qos_power_init);
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 06efa54f93d6..cbe2c1441392 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -1339,6 +1339,9 @@ int hibernate_preallocate_memory(void)
1339 count += highmem; 1339 count += highmem;
1340 count -= totalreserve_pages; 1340 count -= totalreserve_pages;
1341 1341
1342 /* Add number of pages required for page keys (s390 only). */
1343 size += page_key_additional_pages(saveable);
1344
1342 /* Compute the maximum number of saveable pages to leave in memory. */ 1345 /* Compute the maximum number of saveable pages to leave in memory. */
1343 max_size = (count - (size + PAGES_FOR_IO)) / 2 1346 max_size = (count - (size + PAGES_FOR_IO)) / 2
1344 - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE); 1347 - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE);
@@ -1662,6 +1665,8 @@ pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1662 buf[j] = memory_bm_next_pfn(bm); 1665 buf[j] = memory_bm_next_pfn(bm);
1663 if (unlikely(buf[j] == BM_END_OF_MAP)) 1666 if (unlikely(buf[j] == BM_END_OF_MAP))
1664 break; 1667 break;
1668 /* Save page key for data page (s390 only). */
1669 page_key_read(buf + j);
1665 } 1670 }
1666} 1671}
1667 1672
@@ -1821,6 +1826,9 @@ static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1821 if (unlikely(buf[j] == BM_END_OF_MAP)) 1826 if (unlikely(buf[j] == BM_END_OF_MAP))
1822 break; 1827 break;
1823 1828
1829 /* Extract and buffer page key for data page (s390 only). */
1830 page_key_memorize(buf + j);
1831
1824 if (memory_bm_pfn_present(bm, buf[j])) 1832 if (memory_bm_pfn_present(bm, buf[j]))
1825 memory_bm_set_bit(bm, buf[j]); 1833 memory_bm_set_bit(bm, buf[j]);
1826 else 1834 else
@@ -2223,6 +2231,11 @@ int snapshot_write_next(struct snapshot_handle *handle)
2223 if (error) 2231 if (error)
2224 return error; 2232 return error;
2225 2233
2234 /* Allocate buffer for page keys. */
2235 error = page_key_alloc(nr_copy_pages);
2236 if (error)
2237 return error;
2238
2226 } else if (handle->cur <= nr_meta_pages + 1) { 2239 } else if (handle->cur <= nr_meta_pages + 1) {
2227 error = unpack_orig_pfns(buffer, &copy_bm); 2240 error = unpack_orig_pfns(buffer, &copy_bm);
2228 if (error) 2241 if (error)
@@ -2243,6 +2256,8 @@ int snapshot_write_next(struct snapshot_handle *handle)
2243 } 2256 }
2244 } else { 2257 } else {
2245 copy_last_highmem_page(); 2258 copy_last_highmem_page();
2259 /* Restore page key for data page (s390 only). */
2260 page_key_write(handle->buffer);
2246 handle->buffer = get_buffer(&orig_bm, &ca); 2261 handle->buffer = get_buffer(&orig_bm, &ca);
2247 if (IS_ERR(handle->buffer)) 2262 if (IS_ERR(handle->buffer))
2248 return PTR_ERR(handle->buffer); 2263 return PTR_ERR(handle->buffer);
@@ -2264,6 +2279,9 @@ int snapshot_write_next(struct snapshot_handle *handle)
2264void snapshot_write_finalize(struct snapshot_handle *handle) 2279void snapshot_write_finalize(struct snapshot_handle *handle)
2265{ 2280{
2266 copy_last_highmem_page(); 2281 copy_last_highmem_page();
2282 /* Restore page key for data page (s390 only). */
2283 page_key_write(handle->buffer);
2284 page_key_free();
2267 /* Free only if we have loaded the image entirely */ 2285 /* Free only if we have loaded the image entirely */
2268 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) { 2286 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) {
2269 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR); 2287 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index b6b71ad2208f..4953dc054c53 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -12,6 +12,7 @@
12#include <linux/delay.h> 12#include <linux/delay.h>
13#include <linux/errno.h> 13#include <linux/errno.h>
14#include <linux/init.h> 14#include <linux/init.h>
15#include <linux/kmod.h>
15#include <linux/console.h> 16#include <linux/console.h>
16#include <linux/cpu.h> 17#include <linux/cpu.h>
17#include <linux/syscalls.h> 18#include <linux/syscalls.h>
@@ -21,6 +22,7 @@
21#include <linux/list.h> 22#include <linux/list.h>
22#include <linux/mm.h> 23#include <linux/mm.h>
23#include <linux/slab.h> 24#include <linux/slab.h>
25#include <linux/export.h>
24#include <linux/suspend.h> 26#include <linux/suspend.h>
25#include <linux/syscore_ops.h> 27#include <linux/syscore_ops.h>
26#include <trace/events/power.h> 28#include <trace/events/power.h>
@@ -104,7 +106,10 @@ static int suspend_prepare(void)
104 goto Finish; 106 goto Finish;
105 107
106 error = suspend_freeze_processes(); 108 error = suspend_freeze_processes();
107 if (!error) 109 if (error) {
110 suspend_stats.failed_freeze++;
111 dpm_save_failed_step(SUSPEND_FREEZE);
112 } else
108 return 0; 113 return 0;
109 114
110 suspend_thaw_processes(); 115 suspend_thaw_processes();
@@ -315,8 +320,16 @@ int enter_state(suspend_state_t state)
315 */ 320 */
316int pm_suspend(suspend_state_t state) 321int pm_suspend(suspend_state_t state)
317{ 322{
318 if (state > PM_SUSPEND_ON && state <= PM_SUSPEND_MAX) 323 int ret;
319 return enter_state(state); 324 if (state > PM_SUSPEND_ON && state < PM_SUSPEND_MAX) {
325 ret = enter_state(state);
326 if (ret) {
327 suspend_stats.fail++;
328 dpm_save_failed_errno(ret);
329 } else
330 suspend_stats.success++;
331 return ret;
332 }
320 return -EINVAL; 333 return -EINVAL;
321} 334}
322EXPORT_SYMBOL(pm_suspend); 335EXPORT_SYMBOL(pm_suspend);
diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index 7c97c3a0eee3..11a594c4ba25 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -27,6 +27,10 @@
27#include <linux/slab.h> 27#include <linux/slab.h>
28#include <linux/lzo.h> 28#include <linux/lzo.h>
29#include <linux/vmalloc.h> 29#include <linux/vmalloc.h>
30#include <linux/cpumask.h>
31#include <linux/atomic.h>
32#include <linux/kthread.h>
33#include <linux/crc32.h>
30 34
31#include "power.h" 35#include "power.h"
32 36
@@ -43,8 +47,7 @@
43 * allocated and populated one at a time, so we only need one memory 47 * allocated and populated one at a time, so we only need one memory
44 * page to set up the entire structure. 48 * page to set up the entire structure.
45 * 49 *
46 * During resume we also only need to use one swap_map_page structure 50 * During resume we pick up all swap_map_page structures into a list.
47 * at a time.
48 */ 51 */
49 52
50#define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1) 53#define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1)
@@ -54,6 +57,11 @@ struct swap_map_page {
54 sector_t next_swap; 57 sector_t next_swap;
55}; 58};
56 59
60struct swap_map_page_list {
61 struct swap_map_page *map;
62 struct swap_map_page_list *next;
63};
64
57/** 65/**
58 * The swap_map_handle structure is used for handling swap in 66 * The swap_map_handle structure is used for handling swap in
59 * a file-alike way 67 * a file-alike way
@@ -61,13 +69,18 @@ struct swap_map_page {
61 69
62struct swap_map_handle { 70struct swap_map_handle {
63 struct swap_map_page *cur; 71 struct swap_map_page *cur;
72 struct swap_map_page_list *maps;
64 sector_t cur_swap; 73 sector_t cur_swap;
65 sector_t first_sector; 74 sector_t first_sector;
66 unsigned int k; 75 unsigned int k;
76 unsigned long nr_free_pages, written;
77 u32 crc32;
67}; 78};
68 79
69struct swsusp_header { 80struct swsusp_header {
70 char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int)]; 81 char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) -
82 sizeof(u32)];
83 u32 crc32;
71 sector_t image; 84 sector_t image;
72 unsigned int flags; /* Flags to pass to the "boot" kernel */ 85 unsigned int flags; /* Flags to pass to the "boot" kernel */
73 char orig_sig[10]; 86 char orig_sig[10];
@@ -199,6 +212,8 @@ static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
199 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10); 212 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10);
200 swsusp_header->image = handle->first_sector; 213 swsusp_header->image = handle->first_sector;
201 swsusp_header->flags = flags; 214 swsusp_header->flags = flags;
215 if (flags & SF_CRC32_MODE)
216 swsusp_header->crc32 = handle->crc32;
202 error = hib_bio_write_page(swsusp_resume_block, 217 error = hib_bio_write_page(swsusp_resume_block,
203 swsusp_header, NULL); 218 swsusp_header, NULL);
204 } else { 219 } else {
@@ -245,6 +260,7 @@ static int swsusp_swap_check(void)
245static int write_page(void *buf, sector_t offset, struct bio **bio_chain) 260static int write_page(void *buf, sector_t offset, struct bio **bio_chain)
246{ 261{
247 void *src; 262 void *src;
263 int ret;
248 264
249 if (!offset) 265 if (!offset)
250 return -ENOSPC; 266 return -ENOSPC;
@@ -254,9 +270,17 @@ static int write_page(void *buf, sector_t offset, struct bio **bio_chain)
254 if (src) { 270 if (src) {
255 copy_page(src, buf); 271 copy_page(src, buf);
256 } else { 272 } else {
257 WARN_ON_ONCE(1); 273 ret = hib_wait_on_bio_chain(bio_chain); /* Free pages */
258 bio_chain = NULL; /* Go synchronous */ 274 if (ret)
259 src = buf; 275 return ret;
276 src = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
277 if (src) {
278 copy_page(src, buf);
279 } else {
280 WARN_ON_ONCE(1);
281 bio_chain = NULL; /* Go synchronous */
282 src = buf;
283 }
260 } 284 }
261 } else { 285 } else {
262 src = buf; 286 src = buf;
@@ -293,6 +317,8 @@ static int get_swap_writer(struct swap_map_handle *handle)
293 goto err_rel; 317 goto err_rel;
294 } 318 }
295 handle->k = 0; 319 handle->k = 0;
320 handle->nr_free_pages = nr_free_pages() >> 1;
321 handle->written = 0;
296 handle->first_sector = handle->cur_swap; 322 handle->first_sector = handle->cur_swap;
297 return 0; 323 return 0;
298err_rel: 324err_rel:
@@ -316,20 +342,23 @@ static int swap_write_page(struct swap_map_handle *handle, void *buf,
316 return error; 342 return error;
317 handle->cur->entries[handle->k++] = offset; 343 handle->cur->entries[handle->k++] = offset;
318 if (handle->k >= MAP_PAGE_ENTRIES) { 344 if (handle->k >= MAP_PAGE_ENTRIES) {
319 error = hib_wait_on_bio_chain(bio_chain);
320 if (error)
321 goto out;
322 offset = alloc_swapdev_block(root_swap); 345 offset = alloc_swapdev_block(root_swap);
323 if (!offset) 346 if (!offset)
324 return -ENOSPC; 347 return -ENOSPC;
325 handle->cur->next_swap = offset; 348 handle->cur->next_swap = offset;
326 error = write_page(handle->cur, handle->cur_swap, NULL); 349 error = write_page(handle->cur, handle->cur_swap, bio_chain);
327 if (error) 350 if (error)
328 goto out; 351 goto out;
329 clear_page(handle->cur); 352 clear_page(handle->cur);
330 handle->cur_swap = offset; 353 handle->cur_swap = offset;
331 handle->k = 0; 354 handle->k = 0;
332 } 355 }
356 if (bio_chain && ++handle->written > handle->nr_free_pages) {
357 error = hib_wait_on_bio_chain(bio_chain);
358 if (error)
359 goto out;
360 handle->written = 0;
361 }
333 out: 362 out:
334 return error; 363 return error;
335} 364}
@@ -372,6 +401,13 @@ static int swap_writer_finish(struct swap_map_handle *handle,
372 LZO_HEADER, PAGE_SIZE) 401 LZO_HEADER, PAGE_SIZE)
373#define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE) 402#define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE)
374 403
404/* Maximum number of threads for compression/decompression. */
405#define LZO_THREADS 3
406
407/* Maximum number of pages for read buffering. */
408#define LZO_READ_PAGES (MAP_PAGE_ENTRIES * 8)
409
410
375/** 411/**
376 * save_image - save the suspend image data 412 * save_image - save the suspend image data
377 */ 413 */
@@ -419,6 +455,92 @@ static int save_image(struct swap_map_handle *handle,
419 return ret; 455 return ret;
420} 456}
421 457
458/**
459 * Structure used for CRC32.
460 */
461struct crc_data {
462 struct task_struct *thr; /* thread */
463 atomic_t ready; /* ready to start flag */
464 atomic_t stop; /* ready to stop flag */
465 unsigned run_threads; /* nr current threads */
466 wait_queue_head_t go; /* start crc update */
467 wait_queue_head_t done; /* crc update done */
468 u32 *crc32; /* points to handle's crc32 */
469 size_t *unc_len[LZO_THREADS]; /* uncompressed lengths */
470 unsigned char *unc[LZO_THREADS]; /* uncompressed data */
471};
472
473/**
474 * CRC32 update function that runs in its own thread.
475 */
476static int crc32_threadfn(void *data)
477{
478 struct crc_data *d = data;
479 unsigned i;
480
481 while (1) {
482 wait_event(d->go, atomic_read(&d->ready) ||
483 kthread_should_stop());
484 if (kthread_should_stop()) {
485 d->thr = NULL;
486 atomic_set(&d->stop, 1);
487 wake_up(&d->done);
488 break;
489 }
490 atomic_set(&d->ready, 0);
491
492 for (i = 0; i < d->run_threads; i++)
493 *d->crc32 = crc32_le(*d->crc32,
494 d->unc[i], *d->unc_len[i]);
495 atomic_set(&d->stop, 1);
496 wake_up(&d->done);
497 }
498 return 0;
499}
500/**
501 * Structure used for LZO data compression.
502 */
503struct cmp_data {
504 struct task_struct *thr; /* thread */
505 atomic_t ready; /* ready to start flag */
506 atomic_t stop; /* ready to stop flag */
507 int ret; /* return code */
508 wait_queue_head_t go; /* start compression */
509 wait_queue_head_t done; /* compression done */
510 size_t unc_len; /* uncompressed length */
511 size_t cmp_len; /* compressed length */
512 unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */
513 unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */
514 unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */
515};
516
517/**
518 * Compression function that runs in its own thread.
519 */
520static int lzo_compress_threadfn(void *data)
521{
522 struct cmp_data *d = data;
523
524 while (1) {
525 wait_event(d->go, atomic_read(&d->ready) ||
526 kthread_should_stop());
527 if (kthread_should_stop()) {
528 d->thr = NULL;
529 d->ret = -1;
530 atomic_set(&d->stop, 1);
531 wake_up(&d->done);
532 break;
533 }
534 atomic_set(&d->ready, 0);
535
536 d->ret = lzo1x_1_compress(d->unc, d->unc_len,
537 d->cmp + LZO_HEADER, &d->cmp_len,
538 d->wrk);
539 atomic_set(&d->stop, 1);
540 wake_up(&d->done);
541 }
542 return 0;
543}
422 544
423/** 545/**
424 * save_image_lzo - Save the suspend image data compressed with LZO. 546 * save_image_lzo - Save the suspend image data compressed with LZO.
@@ -437,42 +559,93 @@ static int save_image_lzo(struct swap_map_handle *handle,
437 struct bio *bio; 559 struct bio *bio;
438 struct timeval start; 560 struct timeval start;
439 struct timeval stop; 561 struct timeval stop;
440 size_t off, unc_len, cmp_len; 562 size_t off;
441 unsigned char *unc, *cmp, *wrk, *page; 563 unsigned thr, run_threads, nr_threads;
564 unsigned char *page = NULL;
565 struct cmp_data *data = NULL;
566 struct crc_data *crc = NULL;
567
568 /*
569 * We'll limit the number of threads for compression to limit memory
570 * footprint.
571 */
572 nr_threads = num_online_cpus() - 1;
573 nr_threads = clamp_val(nr_threads, 1, LZO_THREADS);
442 574
443 page = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH); 575 page = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
444 if (!page) { 576 if (!page) {
445 printk(KERN_ERR "PM: Failed to allocate LZO page\n"); 577 printk(KERN_ERR "PM: Failed to allocate LZO page\n");
446 return -ENOMEM; 578 ret = -ENOMEM;
579 goto out_clean;
447 } 580 }
448 581
449 wrk = vmalloc(LZO1X_1_MEM_COMPRESS); 582 data = vmalloc(sizeof(*data) * nr_threads);
450 if (!wrk) { 583 if (!data) {
451 printk(KERN_ERR "PM: Failed to allocate LZO workspace\n"); 584 printk(KERN_ERR "PM: Failed to allocate LZO data\n");
452 free_page((unsigned long)page); 585 ret = -ENOMEM;
453 return -ENOMEM; 586 goto out_clean;
454 } 587 }
588 for (thr = 0; thr < nr_threads; thr++)
589 memset(&data[thr], 0, offsetof(struct cmp_data, go));
455 590
456 unc = vmalloc(LZO_UNC_SIZE); 591 crc = kmalloc(sizeof(*crc), GFP_KERNEL);
457 if (!unc) { 592 if (!crc) {
458 printk(KERN_ERR "PM: Failed to allocate LZO uncompressed\n"); 593 printk(KERN_ERR "PM: Failed to allocate crc\n");
459 vfree(wrk); 594 ret = -ENOMEM;
460 free_page((unsigned long)page); 595 goto out_clean;
461 return -ENOMEM; 596 }
597 memset(crc, 0, offsetof(struct crc_data, go));
598
599 /*
600 * Start the compression threads.
601 */
602 for (thr = 0; thr < nr_threads; thr++) {
603 init_waitqueue_head(&data[thr].go);
604 init_waitqueue_head(&data[thr].done);
605
606 data[thr].thr = kthread_run(lzo_compress_threadfn,
607 &data[thr],
608 "image_compress/%u", thr);
609 if (IS_ERR(data[thr].thr)) {
610 data[thr].thr = NULL;
611 printk(KERN_ERR
612 "PM: Cannot start compression threads\n");
613 ret = -ENOMEM;
614 goto out_clean;
615 }
462 } 616 }
463 617
464 cmp = vmalloc(LZO_CMP_SIZE); 618 /*
465 if (!cmp) { 619 * Adjust number of free pages after all allocations have been done.
466 printk(KERN_ERR "PM: Failed to allocate LZO compressed\n"); 620 * We don't want to run out of pages when writing.
467 vfree(unc); 621 */
468 vfree(wrk); 622 handle->nr_free_pages = nr_free_pages() >> 1;
469 free_page((unsigned long)page); 623
470 return -ENOMEM; 624 /*
625 * Start the CRC32 thread.
626 */
627 init_waitqueue_head(&crc->go);
628 init_waitqueue_head(&crc->done);
629
630 handle->crc32 = 0;
631 crc->crc32 = &handle->crc32;
632 for (thr = 0; thr < nr_threads; thr++) {
633 crc->unc[thr] = data[thr].unc;
634 crc->unc_len[thr] = &data[thr].unc_len;
635 }
636
637 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
638 if (IS_ERR(crc->thr)) {
639 crc->thr = NULL;
640 printk(KERN_ERR "PM: Cannot start CRC32 thread\n");
641 ret = -ENOMEM;
642 goto out_clean;
471 } 643 }
472 644
473 printk(KERN_INFO 645 printk(KERN_INFO
646 "PM: Using %u thread(s) for compression.\n"
474 "PM: Compressing and saving image data (%u pages) ... ", 647 "PM: Compressing and saving image data (%u pages) ... ",
475 nr_to_write); 648 nr_threads, nr_to_write);
476 m = nr_to_write / 100; 649 m = nr_to_write / 100;
477 if (!m) 650 if (!m)
478 m = 1; 651 m = 1;
@@ -480,55 +653,83 @@ static int save_image_lzo(struct swap_map_handle *handle,
480 bio = NULL; 653 bio = NULL;
481 do_gettimeofday(&start); 654 do_gettimeofday(&start);
482 for (;;) { 655 for (;;) {
483 for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) { 656 for (thr = 0; thr < nr_threads; thr++) {
484 ret = snapshot_read_next(snapshot); 657 for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) {
485 if (ret < 0) 658 ret = snapshot_read_next(snapshot);
486 goto out_finish; 659 if (ret < 0)
487 660 goto out_finish;
488 if (!ret) 661
662 if (!ret)
663 break;
664
665 memcpy(data[thr].unc + off,
666 data_of(*snapshot), PAGE_SIZE);
667
668 if (!(nr_pages % m))
669 printk(KERN_CONT "\b\b\b\b%3d%%",
670 nr_pages / m);
671 nr_pages++;
672 }
673 if (!off)
489 break; 674 break;
490 675
491 memcpy(unc + off, data_of(*snapshot), PAGE_SIZE); 676 data[thr].unc_len = off;
492 677
493 if (!(nr_pages % m)) 678 atomic_set(&data[thr].ready, 1);
494 printk(KERN_CONT "\b\b\b\b%3d%%", nr_pages / m); 679 wake_up(&data[thr].go);
495 nr_pages++;
496 } 680 }
497 681
498 if (!off) 682 if (!thr)
499 break; 683 break;
500 684
501 unc_len = off; 685 crc->run_threads = thr;
502 ret = lzo1x_1_compress(unc, unc_len, 686 atomic_set(&crc->ready, 1);
503 cmp + LZO_HEADER, &cmp_len, wrk); 687 wake_up(&crc->go);
504 if (ret < 0) {
505 printk(KERN_ERR "PM: LZO compression failed\n");
506 break;
507 }
508 688
509 if (unlikely(!cmp_len || 689 for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
510 cmp_len > lzo1x_worst_compress(unc_len))) { 690 wait_event(data[thr].done,
511 printk(KERN_ERR "PM: Invalid LZO compressed length\n"); 691 atomic_read(&data[thr].stop));
512 ret = -1; 692 atomic_set(&data[thr].stop, 0);
513 break;
514 }
515 693
516 *(size_t *)cmp = cmp_len; 694 ret = data[thr].ret;
517 695
518 /* 696 if (ret < 0) {
519 * Given we are writing one page at a time to disk, we copy 697 printk(KERN_ERR "PM: LZO compression failed\n");
520 * that much from the buffer, although the last bit will likely 698 goto out_finish;
521 * be smaller than full page. This is OK - we saved the length 699 }
522 * of the compressed data, so any garbage at the end will be
523 * discarded when we read it.
524 */
525 for (off = 0; off < LZO_HEADER + cmp_len; off += PAGE_SIZE) {
526 memcpy(page, cmp + off, PAGE_SIZE);
527 700
528 ret = swap_write_page(handle, page, &bio); 701 if (unlikely(!data[thr].cmp_len ||
529 if (ret) 702 data[thr].cmp_len >
703 lzo1x_worst_compress(data[thr].unc_len))) {
704 printk(KERN_ERR
705 "PM: Invalid LZO compressed length\n");
706 ret = -1;
530 goto out_finish; 707 goto out_finish;
708 }
709
710 *(size_t *)data[thr].cmp = data[thr].cmp_len;
711
712 /*
713 * Given we are writing one page at a time to disk, we
714 * copy that much from the buffer, although the last
715 * bit will likely be smaller than full page. This is
716 * OK - we saved the length of the compressed data, so
717 * any garbage at the end will be discarded when we
718 * read it.
719 */
720 for (off = 0;
721 off < LZO_HEADER + data[thr].cmp_len;
722 off += PAGE_SIZE) {
723 memcpy(page, data[thr].cmp + off, PAGE_SIZE);
724
725 ret = swap_write_page(handle, page, &bio);
726 if (ret)
727 goto out_finish;
728 }
531 } 729 }
730
731 wait_event(crc->done, atomic_read(&crc->stop));
732 atomic_set(&crc->stop, 0);
532 } 733 }
533 734
534out_finish: 735out_finish:
@@ -536,16 +737,25 @@ out_finish:
536 do_gettimeofday(&stop); 737 do_gettimeofday(&stop);
537 if (!ret) 738 if (!ret)
538 ret = err2; 739 ret = err2;
539 if (!ret) 740 if (!ret) {
540 printk(KERN_CONT "\b\b\b\bdone\n"); 741 printk(KERN_CONT "\b\b\b\bdone\n");
541 else 742 } else {
542 printk(KERN_CONT "\n"); 743 printk(KERN_CONT "\n");
744 }
543 swsusp_show_speed(&start, &stop, nr_to_write, "Wrote"); 745 swsusp_show_speed(&start, &stop, nr_to_write, "Wrote");
544 746out_clean:
545 vfree(cmp); 747 if (crc) {
546 vfree(unc); 748 if (crc->thr)
547 vfree(wrk); 749 kthread_stop(crc->thr);
548 free_page((unsigned long)page); 750 kfree(crc);
751 }
752 if (data) {
753 for (thr = 0; thr < nr_threads; thr++)
754 if (data[thr].thr)
755 kthread_stop(data[thr].thr);
756 vfree(data);
757 }
758 if (page) free_page((unsigned long)page);
549 759
550 return ret; 760 return ret;
551} 761}
@@ -625,8 +835,15 @@ out_finish:
625 835
626static void release_swap_reader(struct swap_map_handle *handle) 836static void release_swap_reader(struct swap_map_handle *handle)
627{ 837{
628 if (handle->cur) 838 struct swap_map_page_list *tmp;
629 free_page((unsigned long)handle->cur); 839
840 while (handle->maps) {
841 if (handle->maps->map)
842 free_page((unsigned long)handle->maps->map);
843 tmp = handle->maps;
844 handle->maps = handle->maps->next;
845 kfree(tmp);
846 }
630 handle->cur = NULL; 847 handle->cur = NULL;
631} 848}
632 849
@@ -634,22 +851,46 @@ static int get_swap_reader(struct swap_map_handle *handle,
634 unsigned int *flags_p) 851 unsigned int *flags_p)
635{ 852{
636 int error; 853 int error;
854 struct swap_map_page_list *tmp, *last;
855 sector_t offset;
637 856
638 *flags_p = swsusp_header->flags; 857 *flags_p = swsusp_header->flags;
639 858
640 if (!swsusp_header->image) /* how can this happen? */ 859 if (!swsusp_header->image) /* how can this happen? */
641 return -EINVAL; 860 return -EINVAL;
642 861
643 handle->cur = (struct swap_map_page *)get_zeroed_page(__GFP_WAIT | __GFP_HIGH); 862 handle->cur = NULL;
644 if (!handle->cur) 863 last = handle->maps = NULL;
645 return -ENOMEM; 864 offset = swsusp_header->image;
865 while (offset) {
866 tmp = kmalloc(sizeof(*handle->maps), GFP_KERNEL);
867 if (!tmp) {
868 release_swap_reader(handle);
869 return -ENOMEM;
870 }
871 memset(tmp, 0, sizeof(*tmp));
872 if (!handle->maps)
873 handle->maps = tmp;
874 if (last)
875 last->next = tmp;
876 last = tmp;
877
878 tmp->map = (struct swap_map_page *)
879 __get_free_page(__GFP_WAIT | __GFP_HIGH);
880 if (!tmp->map) {
881 release_swap_reader(handle);
882 return -ENOMEM;
883 }
646 884
647 error = hib_bio_read_page(swsusp_header->image, handle->cur, NULL); 885 error = hib_bio_read_page(offset, tmp->map, NULL);
648 if (error) { 886 if (error) {
649 release_swap_reader(handle); 887 release_swap_reader(handle);
650 return error; 888 return error;
889 }
890 offset = tmp->map->next_swap;
651 } 891 }
652 handle->k = 0; 892 handle->k = 0;
893 handle->cur = handle->maps->map;
653 return 0; 894 return 0;
654} 895}
655 896
@@ -658,6 +899,7 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf,
658{ 899{
659 sector_t offset; 900 sector_t offset;
660 int error; 901 int error;
902 struct swap_map_page_list *tmp;
661 903
662 if (!handle->cur) 904 if (!handle->cur)
663 return -EINVAL; 905 return -EINVAL;
@@ -668,13 +910,15 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf,
668 if (error) 910 if (error)
669 return error; 911 return error;
670 if (++handle->k >= MAP_PAGE_ENTRIES) { 912 if (++handle->k >= MAP_PAGE_ENTRIES) {
671 error = hib_wait_on_bio_chain(bio_chain);
672 handle->k = 0; 913 handle->k = 0;
673 offset = handle->cur->next_swap; 914 free_page((unsigned long)handle->maps->map);
674 if (!offset) 915 tmp = handle->maps;
916 handle->maps = handle->maps->next;
917 kfree(tmp);
918 if (!handle->maps)
675 release_swap_reader(handle); 919 release_swap_reader(handle);
676 else if (!error) 920 else
677 error = hib_bio_read_page(offset, handle->cur, NULL); 921 handle->cur = handle->maps->map;
678 } 922 }
679 return error; 923 return error;
680} 924}
@@ -697,7 +941,7 @@ static int load_image(struct swap_map_handle *handle,
697 unsigned int nr_to_read) 941 unsigned int nr_to_read)
698{ 942{
699 unsigned int m; 943 unsigned int m;
700 int error = 0; 944 int ret = 0;
701 struct timeval start; 945 struct timeval start;
702 struct timeval stop; 946 struct timeval stop;
703 struct bio *bio; 947 struct bio *bio;
@@ -713,15 +957,15 @@ static int load_image(struct swap_map_handle *handle,
713 bio = NULL; 957 bio = NULL;
714 do_gettimeofday(&start); 958 do_gettimeofday(&start);
715 for ( ; ; ) { 959 for ( ; ; ) {
716 error = snapshot_write_next(snapshot); 960 ret = snapshot_write_next(snapshot);
717 if (error <= 0) 961 if (ret <= 0)
718 break; 962 break;
719 error = swap_read_page(handle, data_of(*snapshot), &bio); 963 ret = swap_read_page(handle, data_of(*snapshot), &bio);
720 if (error) 964 if (ret)
721 break; 965 break;
722 if (snapshot->sync_read) 966 if (snapshot->sync_read)
723 error = hib_wait_on_bio_chain(&bio); 967 ret = hib_wait_on_bio_chain(&bio);
724 if (error) 968 if (ret)
725 break; 969 break;
726 if (!(nr_pages % m)) 970 if (!(nr_pages % m))
727 printk("\b\b\b\b%3d%%", nr_pages / m); 971 printk("\b\b\b\b%3d%%", nr_pages / m);
@@ -729,17 +973,61 @@ static int load_image(struct swap_map_handle *handle,
729 } 973 }
730 err2 = hib_wait_on_bio_chain(&bio); 974 err2 = hib_wait_on_bio_chain(&bio);
731 do_gettimeofday(&stop); 975 do_gettimeofday(&stop);
732 if (!error) 976 if (!ret)
733 error = err2; 977 ret = err2;
734 if (!error) { 978 if (!ret) {
735 printk("\b\b\b\bdone\n"); 979 printk("\b\b\b\bdone\n");
736 snapshot_write_finalize(snapshot); 980 snapshot_write_finalize(snapshot);
737 if (!snapshot_image_loaded(snapshot)) 981 if (!snapshot_image_loaded(snapshot))
738 error = -ENODATA; 982 ret = -ENODATA;
739 } else 983 } else
740 printk("\n"); 984 printk("\n");
741 swsusp_show_speed(&start, &stop, nr_to_read, "Read"); 985 swsusp_show_speed(&start, &stop, nr_to_read, "Read");
742 return error; 986 return ret;
987}
988
989/**
990 * Structure used for LZO data decompression.
991 */
992struct dec_data {
993 struct task_struct *thr; /* thread */
994 atomic_t ready; /* ready to start flag */
995 atomic_t stop; /* ready to stop flag */
996 int ret; /* return code */
997 wait_queue_head_t go; /* start decompression */
998 wait_queue_head_t done; /* decompression done */
999 size_t unc_len; /* uncompressed length */
1000 size_t cmp_len; /* compressed length */
1001 unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */
1002 unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */
1003};
1004
1005/**
1006 * Deompression function that runs in its own thread.
1007 */
1008static int lzo_decompress_threadfn(void *data)
1009{
1010 struct dec_data *d = data;
1011
1012 while (1) {
1013 wait_event(d->go, atomic_read(&d->ready) ||
1014 kthread_should_stop());
1015 if (kthread_should_stop()) {
1016 d->thr = NULL;
1017 d->ret = -1;
1018 atomic_set(&d->stop, 1);
1019 wake_up(&d->done);
1020 break;
1021 }
1022 atomic_set(&d->ready, 0);
1023
1024 d->unc_len = LZO_UNC_SIZE;
1025 d->ret = lzo1x_decompress_safe(d->cmp + LZO_HEADER, d->cmp_len,
1026 d->unc, &d->unc_len);
1027 atomic_set(&d->stop, 1);
1028 wake_up(&d->done);
1029 }
1030 return 0;
743} 1031}
744 1032
745/** 1033/**
@@ -753,50 +1041,120 @@ static int load_image_lzo(struct swap_map_handle *handle,
753 unsigned int nr_to_read) 1041 unsigned int nr_to_read)
754{ 1042{
755 unsigned int m; 1043 unsigned int m;
756 int error = 0; 1044 int ret = 0;
1045 int eof = 0;
757 struct bio *bio; 1046 struct bio *bio;
758 struct timeval start; 1047 struct timeval start;
759 struct timeval stop; 1048 struct timeval stop;
760 unsigned nr_pages; 1049 unsigned nr_pages;
761 size_t i, off, unc_len, cmp_len; 1050 size_t off;
762 unsigned char *unc, *cmp, *page[LZO_CMP_PAGES]; 1051 unsigned i, thr, run_threads, nr_threads;
763 1052 unsigned ring = 0, pg = 0, ring_size = 0,
764 for (i = 0; i < LZO_CMP_PAGES; i++) { 1053 have = 0, want, need, asked = 0;
765 page[i] = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH); 1054 unsigned long read_pages;
766 if (!page[i]) { 1055 unsigned char **page = NULL;
767 printk(KERN_ERR "PM: Failed to allocate LZO page\n"); 1056 struct dec_data *data = NULL;
1057 struct crc_data *crc = NULL;
1058
1059 /*
1060 * We'll limit the number of threads for decompression to limit memory
1061 * footprint.
1062 */
1063 nr_threads = num_online_cpus() - 1;
1064 nr_threads = clamp_val(nr_threads, 1, LZO_THREADS);
1065
1066 page = vmalloc(sizeof(*page) * LZO_READ_PAGES);
1067 if (!page) {
1068 printk(KERN_ERR "PM: Failed to allocate LZO page\n");
1069 ret = -ENOMEM;
1070 goto out_clean;
1071 }
768 1072
769 while (i) 1073 data = vmalloc(sizeof(*data) * nr_threads);
770 free_page((unsigned long)page[--i]); 1074 if (!data) {
1075 printk(KERN_ERR "PM: Failed to allocate LZO data\n");
1076 ret = -ENOMEM;
1077 goto out_clean;
1078 }
1079 for (thr = 0; thr < nr_threads; thr++)
1080 memset(&data[thr], 0, offsetof(struct dec_data, go));
771 1081
772 return -ENOMEM; 1082 crc = kmalloc(sizeof(*crc), GFP_KERNEL);
1083 if (!crc) {
1084 printk(KERN_ERR "PM: Failed to allocate crc\n");
1085 ret = -ENOMEM;
1086 goto out_clean;
1087 }
1088 memset(crc, 0, offsetof(struct crc_data, go));
1089
1090 /*
1091 * Start the decompression threads.
1092 */
1093 for (thr = 0; thr < nr_threads; thr++) {
1094 init_waitqueue_head(&data[thr].go);
1095 init_waitqueue_head(&data[thr].done);
1096
1097 data[thr].thr = kthread_run(lzo_decompress_threadfn,
1098 &data[thr],
1099 "image_decompress/%u", thr);
1100 if (IS_ERR(data[thr].thr)) {
1101 data[thr].thr = NULL;
1102 printk(KERN_ERR
1103 "PM: Cannot start decompression threads\n");
1104 ret = -ENOMEM;
1105 goto out_clean;
773 } 1106 }
774 } 1107 }
775 1108
776 unc = vmalloc(LZO_UNC_SIZE); 1109 /*
777 if (!unc) { 1110 * Start the CRC32 thread.
778 printk(KERN_ERR "PM: Failed to allocate LZO uncompressed\n"); 1111 */
779 1112 init_waitqueue_head(&crc->go);
780 for (i = 0; i < LZO_CMP_PAGES; i++) 1113 init_waitqueue_head(&crc->done);
781 free_page((unsigned long)page[i]); 1114
782 1115 handle->crc32 = 0;
783 return -ENOMEM; 1116 crc->crc32 = &handle->crc32;
1117 for (thr = 0; thr < nr_threads; thr++) {
1118 crc->unc[thr] = data[thr].unc;
1119 crc->unc_len[thr] = &data[thr].unc_len;
784 } 1120 }
785 1121
786 cmp = vmalloc(LZO_CMP_SIZE); 1122 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
787 if (!cmp) { 1123 if (IS_ERR(crc->thr)) {
788 printk(KERN_ERR "PM: Failed to allocate LZO compressed\n"); 1124 crc->thr = NULL;
1125 printk(KERN_ERR "PM: Cannot start CRC32 thread\n");
1126 ret = -ENOMEM;
1127 goto out_clean;
1128 }
789 1129
790 vfree(unc); 1130 /*
791 for (i = 0; i < LZO_CMP_PAGES; i++) 1131 * Adjust number of pages for read buffering, in case we are short.
792 free_page((unsigned long)page[i]); 1132 */
1133 read_pages = (nr_free_pages() - snapshot_get_image_size()) >> 1;
1134 read_pages = clamp_val(read_pages, LZO_CMP_PAGES, LZO_READ_PAGES);
793 1135
794 return -ENOMEM; 1136 for (i = 0; i < read_pages; i++) {
1137 page[i] = (void *)__get_free_page(i < LZO_CMP_PAGES ?
1138 __GFP_WAIT | __GFP_HIGH :
1139 __GFP_WAIT);
1140 if (!page[i]) {
1141 if (i < LZO_CMP_PAGES) {
1142 ring_size = i;
1143 printk(KERN_ERR
1144 "PM: Failed to allocate LZO pages\n");
1145 ret = -ENOMEM;
1146 goto out_clean;
1147 } else {
1148 break;
1149 }
1150 }
795 } 1151 }
1152 want = ring_size = i;
796 1153
797 printk(KERN_INFO 1154 printk(KERN_INFO
1155 "PM: Using %u thread(s) for decompression.\n"
798 "PM: Loading and decompressing image data (%u pages) ... ", 1156 "PM: Loading and decompressing image data (%u pages) ... ",
799 nr_to_read); 1157 nr_threads, nr_to_read);
800 m = nr_to_read / 100; 1158 m = nr_to_read / 100;
801 if (!m) 1159 if (!m)
802 m = 1; 1160 m = 1;
@@ -804,85 +1162,189 @@ static int load_image_lzo(struct swap_map_handle *handle,
804 bio = NULL; 1162 bio = NULL;
805 do_gettimeofday(&start); 1163 do_gettimeofday(&start);
806 1164
807 error = snapshot_write_next(snapshot); 1165 ret = snapshot_write_next(snapshot);
808 if (error <= 0) 1166 if (ret <= 0)
809 goto out_finish; 1167 goto out_finish;
810 1168
811 for (;;) { 1169 for(;;) {
812 error = swap_read_page(handle, page[0], NULL); /* sync */ 1170 for (i = 0; !eof && i < want; i++) {
813 if (error) 1171 ret = swap_read_page(handle, page[ring], &bio);
814 break; 1172 if (ret) {
815 1173 /*
816 cmp_len = *(size_t *)page[0]; 1174 * On real read error, finish. On end of data,
817 if (unlikely(!cmp_len || 1175 * set EOF flag and just exit the read loop.
818 cmp_len > lzo1x_worst_compress(LZO_UNC_SIZE))) { 1176 */
819 printk(KERN_ERR "PM: Invalid LZO compressed length\n"); 1177 if (handle->cur &&
820 error = -1; 1178 handle->cur->entries[handle->k]) {
821 break; 1179 goto out_finish;
1180 } else {
1181 eof = 1;
1182 break;
1183 }
1184 }
1185 if (++ring >= ring_size)
1186 ring = 0;
822 } 1187 }
1188 asked += i;
1189 want -= i;
823 1190
824 for (off = PAGE_SIZE, i = 1; 1191 /*
825 off < LZO_HEADER + cmp_len; off += PAGE_SIZE, i++) { 1192 * We are out of data, wait for some more.
826 error = swap_read_page(handle, page[i], &bio); 1193 */
827 if (error) 1194 if (!have) {
1195 if (!asked)
1196 break;
1197
1198 ret = hib_wait_on_bio_chain(&bio);
1199 if (ret)
828 goto out_finish; 1200 goto out_finish;
1201 have += asked;
1202 asked = 0;
1203 if (eof)
1204 eof = 2;
829 } 1205 }
830 1206
831 error = hib_wait_on_bio_chain(&bio); /* need all data now */ 1207 if (crc->run_threads) {
832 if (error) 1208 wait_event(crc->done, atomic_read(&crc->stop));
833 goto out_finish; 1209 atomic_set(&crc->stop, 0);
834 1210 crc->run_threads = 0;
835 for (off = 0, i = 0;
836 off < LZO_HEADER + cmp_len; off += PAGE_SIZE, i++) {
837 memcpy(cmp + off, page[i], PAGE_SIZE);
838 } 1211 }
839 1212
840 unc_len = LZO_UNC_SIZE; 1213 for (thr = 0; have && thr < nr_threads; thr++) {
841 error = lzo1x_decompress_safe(cmp + LZO_HEADER, cmp_len, 1214 data[thr].cmp_len = *(size_t *)page[pg];
842 unc, &unc_len); 1215 if (unlikely(!data[thr].cmp_len ||
843 if (error < 0) { 1216 data[thr].cmp_len >
844 printk(KERN_ERR "PM: LZO decompression failed\n"); 1217 lzo1x_worst_compress(LZO_UNC_SIZE))) {
845 break; 1218 printk(KERN_ERR
1219 "PM: Invalid LZO compressed length\n");
1220 ret = -1;
1221 goto out_finish;
1222 }
1223
1224 need = DIV_ROUND_UP(data[thr].cmp_len + LZO_HEADER,
1225 PAGE_SIZE);
1226 if (need > have) {
1227 if (eof > 1) {
1228 ret = -1;
1229 goto out_finish;
1230 }
1231 break;
1232 }
1233
1234 for (off = 0;
1235 off < LZO_HEADER + data[thr].cmp_len;
1236 off += PAGE_SIZE) {
1237 memcpy(data[thr].cmp + off,
1238 page[pg], PAGE_SIZE);
1239 have--;
1240 want++;
1241 if (++pg >= ring_size)
1242 pg = 0;
1243 }
1244
1245 atomic_set(&data[thr].ready, 1);
1246 wake_up(&data[thr].go);
846 } 1247 }
847 1248
848 if (unlikely(!unc_len || 1249 /*
849 unc_len > LZO_UNC_SIZE || 1250 * Wait for more data while we are decompressing.
850 unc_len & (PAGE_SIZE - 1))) { 1251 */
851 printk(KERN_ERR "PM: Invalid LZO uncompressed length\n"); 1252 if (have < LZO_CMP_PAGES && asked) {
852 error = -1; 1253 ret = hib_wait_on_bio_chain(&bio);
853 break; 1254 if (ret)
1255 goto out_finish;
1256 have += asked;
1257 asked = 0;
1258 if (eof)
1259 eof = 2;
854 } 1260 }
855 1261
856 for (off = 0; off < unc_len; off += PAGE_SIZE) { 1262 for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
857 memcpy(data_of(*snapshot), unc + off, PAGE_SIZE); 1263 wait_event(data[thr].done,
1264 atomic_read(&data[thr].stop));
1265 atomic_set(&data[thr].stop, 0);
1266
1267 ret = data[thr].ret;
858 1268
859 if (!(nr_pages % m)) 1269 if (ret < 0) {
860 printk("\b\b\b\b%3d%%", nr_pages / m); 1270 printk(KERN_ERR
861 nr_pages++; 1271 "PM: LZO decompression failed\n");
1272 goto out_finish;
1273 }
862 1274
863 error = snapshot_write_next(snapshot); 1275 if (unlikely(!data[thr].unc_len ||
864 if (error <= 0) 1276 data[thr].unc_len > LZO_UNC_SIZE ||
1277 data[thr].unc_len & (PAGE_SIZE - 1))) {
1278 printk(KERN_ERR
1279 "PM: Invalid LZO uncompressed length\n");
1280 ret = -1;
865 goto out_finish; 1281 goto out_finish;
1282 }
1283
1284 for (off = 0;
1285 off < data[thr].unc_len; off += PAGE_SIZE) {
1286 memcpy(data_of(*snapshot),
1287 data[thr].unc + off, PAGE_SIZE);
1288
1289 if (!(nr_pages % m))
1290 printk("\b\b\b\b%3d%%", nr_pages / m);
1291 nr_pages++;
1292
1293 ret = snapshot_write_next(snapshot);
1294 if (ret <= 0) {
1295 crc->run_threads = thr + 1;
1296 atomic_set(&crc->ready, 1);
1297 wake_up(&crc->go);
1298 goto out_finish;
1299 }
1300 }
866 } 1301 }
1302
1303 crc->run_threads = thr;
1304 atomic_set(&crc->ready, 1);
1305 wake_up(&crc->go);
867 } 1306 }
868 1307
869out_finish: 1308out_finish:
1309 if (crc->run_threads) {
1310 wait_event(crc->done, atomic_read(&crc->stop));
1311 atomic_set(&crc->stop, 0);
1312 }
870 do_gettimeofday(&stop); 1313 do_gettimeofday(&stop);
871 if (!error) { 1314 if (!ret) {
872 printk("\b\b\b\bdone\n"); 1315 printk("\b\b\b\bdone\n");
873 snapshot_write_finalize(snapshot); 1316 snapshot_write_finalize(snapshot);
874 if (!snapshot_image_loaded(snapshot)) 1317 if (!snapshot_image_loaded(snapshot))
875 error = -ENODATA; 1318 ret = -ENODATA;
1319 if (!ret) {
1320 if (swsusp_header->flags & SF_CRC32_MODE) {
1321 if(handle->crc32 != swsusp_header->crc32) {
1322 printk(KERN_ERR
1323 "PM: Invalid image CRC32!\n");
1324 ret = -ENODATA;
1325 }
1326 }
1327 }
876 } else 1328 } else
877 printk("\n"); 1329 printk("\n");
878 swsusp_show_speed(&start, &stop, nr_to_read, "Read"); 1330 swsusp_show_speed(&start, &stop, nr_to_read, "Read");
879 1331out_clean:
880 vfree(cmp); 1332 for (i = 0; i < ring_size; i++)
881 vfree(unc);
882 for (i = 0; i < LZO_CMP_PAGES; i++)
883 free_page((unsigned long)page[i]); 1333 free_page((unsigned long)page[i]);
1334 if (crc) {
1335 if (crc->thr)
1336 kthread_stop(crc->thr);
1337 kfree(crc);
1338 }
1339 if (data) {
1340 for (thr = 0; thr < nr_threads; thr++)
1341 if (data[thr].thr)
1342 kthread_stop(data[thr].thr);
1343 vfree(data);
1344 }
1345 if (page) vfree(page);
884 1346
885 return error; 1347 return ret;
886} 1348}
887 1349
888/** 1350/**
diff --git a/kernel/power/user.c b/kernel/power/user.c
index 42ddbc6f0de6..6d8f535c2b88 100644
--- a/kernel/power/user.c
+++ b/kernel/power/user.c
@@ -12,6 +12,7 @@
12#include <linux/suspend.h> 12#include <linux/suspend.h>
13#include <linux/syscalls.h> 13#include <linux/syscalls.h>
14#include <linux/reboot.h> 14#include <linux/reboot.h>
15#include <linux/kmod.h>
15#include <linux/string.h> 16#include <linux/string.h>
16#include <linux/device.h> 17#include <linux/device.h>
17#include <linux/miscdevice.h> 18#include <linux/miscdevice.h>
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-12 03:46:59 -0400