aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/s390/Kconfig2
-rw-r--r--arch/s390/crypto/crypt_s390.h122
-rw-r--r--arch/s390/crypto/prng.c850
-rw-r--r--arch/s390/include/asm/kexec.h3
-rw-r--r--arch/s390/include/asm/mmu.h4
-rw-r--r--arch/s390/include/asm/mmu_context.h3
-rw-r--r--arch/s390/include/asm/pgalloc.h1
-rw-r--r--arch/s390/include/asm/pgtable.h167
-rw-r--r--arch/s390/mm/hugetlbpage.c66
-rw-r--r--arch/s390/mm/pgtable.c142
-rw-r--r--drivers/s390/char/con3215.c2
-rw-r--r--include/linux/kexec.h4
-rw-r--r--kernel/kexec.c2
13 files changed, 1039 insertions, 329 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 8e58c614c37d..b06dc3839268 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -115,7 +115,7 @@ config S390
115 select HAVE_ARCH_SECCOMP_FILTER 115 select HAVE_ARCH_SECCOMP_FILTER
116 select HAVE_ARCH_TRACEHOOK 116 select HAVE_ARCH_TRACEHOOK
117 select HAVE_ARCH_TRANSPARENT_HUGEPAGE 117 select HAVE_ARCH_TRANSPARENT_HUGEPAGE
118 select HAVE_BPF_JIT if PACK_STACK && HAVE_MARCH_Z9_109_FEATURES 118 select HAVE_BPF_JIT if PACK_STACK && HAVE_MARCH_Z196_FEATURES
119 select HAVE_CMPXCHG_DOUBLE 119 select HAVE_CMPXCHG_DOUBLE
120 select HAVE_CMPXCHG_LOCAL 120 select HAVE_CMPXCHG_LOCAL
121 select HAVE_DEBUG_KMEMLEAK 121 select HAVE_DEBUG_KMEMLEAK
diff --git a/arch/s390/crypto/crypt_s390.h b/arch/s390/crypto/crypt_s390.h
index ba3b2aefddf5..d9c4c313fbc6 100644
--- a/arch/s390/crypto/crypt_s390.h
+++ b/arch/s390/crypto/crypt_s390.h
@@ -3,9 +3,10 @@
3 * 3 *
4 * Support for s390 cryptographic instructions. 4 * Support for s390 cryptographic instructions.
5 * 5 *
6 * Copyright IBM Corp. 2003, 2007 6 * Copyright IBM Corp. 2003, 2015
7 * Author(s): Thomas Spatzier 7 * Author(s): Thomas Spatzier
8 * Jan Glauber (jan.glauber@de.ibm.com) 8 * Jan Glauber (jan.glauber@de.ibm.com)
9 * Harald Freudenberger (freude@de.ibm.com)
9 * 10 *
10 * This program is free software; you can redistribute it and/or modify it 11 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free 12 * under the terms of the GNU General Public License as published by the Free
@@ -28,15 +29,17 @@
28#define CRYPT_S390_MSA 0x1 29#define CRYPT_S390_MSA 0x1
29#define CRYPT_S390_MSA3 0x2 30#define CRYPT_S390_MSA3 0x2
30#define CRYPT_S390_MSA4 0x4 31#define CRYPT_S390_MSA4 0x4
32#define CRYPT_S390_MSA5 0x8
31 33
32/* s390 cryptographic operations */ 34/* s390 cryptographic operations */
33enum crypt_s390_operations { 35enum crypt_s390_operations {
34 CRYPT_S390_KM = 0x0100, 36 CRYPT_S390_KM = 0x0100,
35 CRYPT_S390_KMC = 0x0200, 37 CRYPT_S390_KMC = 0x0200,
36 CRYPT_S390_KIMD = 0x0300, 38 CRYPT_S390_KIMD = 0x0300,
37 CRYPT_S390_KLMD = 0x0400, 39 CRYPT_S390_KLMD = 0x0400,
38 CRYPT_S390_KMAC = 0x0500, 40 CRYPT_S390_KMAC = 0x0500,
39 CRYPT_S390_KMCTR = 0x0600 41 CRYPT_S390_KMCTR = 0x0600,
42 CRYPT_S390_PPNO = 0x0700
40}; 43};
41 44
42/* 45/*
@@ -138,6 +141,16 @@ enum crypt_s390_kmac_func {
138 KMAC_TDEA_192 = CRYPT_S390_KMAC | 3 141 KMAC_TDEA_192 = CRYPT_S390_KMAC | 3
139}; 142};
140 143
144/*
145 * function codes for PPNO (PERFORM PSEUDORANDOM NUMBER
146 * OPERATION) instruction
147 */
148enum crypt_s390_ppno_func {
149 PPNO_QUERY = CRYPT_S390_PPNO | 0,
150 PPNO_SHA512_DRNG_GEN = CRYPT_S390_PPNO | 3,
151 PPNO_SHA512_DRNG_SEED = CRYPT_S390_PPNO | 0x83
152};
153
141/** 154/**
142 * crypt_s390_km: 155 * crypt_s390_km:
143 * @func: the function code passed to KM; see crypt_s390_km_func 156 * @func: the function code passed to KM; see crypt_s390_km_func
@@ -162,11 +175,11 @@ static inline int crypt_s390_km(long func, void *param,
162 int ret; 175 int ret;
163 176
164 asm volatile( 177 asm volatile(
165 "0: .insn rre,0xb92e0000,%3,%1 \n" /* KM opcode */ 178 "0: .insn rre,0xb92e0000,%3,%1\n" /* KM opcode */
166 "1: brc 1,0b \n" /* handle partial completion */ 179 "1: brc 1,0b\n" /* handle partial completion */
167 " la %0,0\n" 180 " la %0,0\n"
168 "2:\n" 181 "2:\n"
169 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 182 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
170 : "=d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest) 183 : "=d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest)
171 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); 184 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory");
172 if (ret < 0) 185 if (ret < 0)
@@ -198,11 +211,11 @@ static inline int crypt_s390_kmc(long func, void *param,
198 int ret; 211 int ret;
199 212
200 asm volatile( 213 asm volatile(
201 "0: .insn rre,0xb92f0000,%3,%1 \n" /* KMC opcode */ 214 "0: .insn rre,0xb92f0000,%3,%1\n" /* KMC opcode */
202 "1: brc 1,0b \n" /* handle partial completion */ 215 "1: brc 1,0b\n" /* handle partial completion */
203 " la %0,0\n" 216 " la %0,0\n"
204 "2:\n" 217 "2:\n"
205 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 218 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
206 : "=d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest) 219 : "=d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest)
207 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); 220 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory");
208 if (ret < 0) 221 if (ret < 0)
@@ -233,11 +246,11 @@ static inline int crypt_s390_kimd(long func, void *param,
233 int ret; 246 int ret;
234 247
235 asm volatile( 248 asm volatile(
236 "0: .insn rre,0xb93e0000,%1,%1 \n" /* KIMD opcode */ 249 "0: .insn rre,0xb93e0000,%1,%1\n" /* KIMD opcode */
237 "1: brc 1,0b \n" /* handle partial completion */ 250 "1: brc 1,0b\n" /* handle partial completion */
238 " la %0,0\n" 251 " la %0,0\n"
239 "2:\n" 252 "2:\n"
240 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 253 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
241 : "=d" (ret), "+a" (__src), "+d" (__src_len) 254 : "=d" (ret), "+a" (__src), "+d" (__src_len)
242 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); 255 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory");
243 if (ret < 0) 256 if (ret < 0)
@@ -267,11 +280,11 @@ static inline int crypt_s390_klmd(long func, void *param,
267 int ret; 280 int ret;
268 281
269 asm volatile( 282 asm volatile(
270 "0: .insn rre,0xb93f0000,%1,%1 \n" /* KLMD opcode */ 283 "0: .insn rre,0xb93f0000,%1,%1\n" /* KLMD opcode */
271 "1: brc 1,0b \n" /* handle partial completion */ 284 "1: brc 1,0b\n" /* handle partial completion */
272 " la %0,0\n" 285 " la %0,0\n"
273 "2:\n" 286 "2:\n"
274 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 287 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
275 : "=d" (ret), "+a" (__src), "+d" (__src_len) 288 : "=d" (ret), "+a" (__src), "+d" (__src_len)
276 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); 289 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory");
277 if (ret < 0) 290 if (ret < 0)
@@ -302,11 +315,11 @@ static inline int crypt_s390_kmac(long func, void *param,
302 int ret; 315 int ret;
303 316
304 asm volatile( 317 asm volatile(
305 "0: .insn rre,0xb91e0000,%1,%1 \n" /* KLAC opcode */ 318 "0: .insn rre,0xb91e0000,%1,%1\n" /* KLAC opcode */
306 "1: brc 1,0b \n" /* handle partial completion */ 319 "1: brc 1,0b\n" /* handle partial completion */
307 " la %0,0\n" 320 " la %0,0\n"
308 "2:\n" 321 "2:\n"
309 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 322 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
310 : "=d" (ret), "+a" (__src), "+d" (__src_len) 323 : "=d" (ret), "+a" (__src), "+d" (__src_len)
311 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory"); 324 : "d" (__func), "a" (__param), "0" (-1) : "cc", "memory");
312 if (ret < 0) 325 if (ret < 0)
@@ -340,11 +353,11 @@ static inline int crypt_s390_kmctr(long func, void *param, u8 *dest,
340 int ret = -1; 353 int ret = -1;
341 354
342 asm volatile( 355 asm volatile(
343 "0: .insn rrf,0xb92d0000,%3,%1,%4,0 \n" /* KMCTR opcode */ 356 "0: .insn rrf,0xb92d0000,%3,%1,%4,0\n" /* KMCTR opcode */
344 "1: brc 1,0b \n" /* handle partial completion */ 357 "1: brc 1,0b\n" /* handle partial completion */
345 " la %0,0\n" 358 " la %0,0\n"
346 "2:\n" 359 "2:\n"
347 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 360 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
348 : "+d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest), 361 : "+d" (ret), "+a" (__src), "+d" (__src_len), "+a" (__dest),
349 "+a" (__ctr) 362 "+a" (__ctr)
350 : "d" (__func), "a" (__param) : "cc", "memory"); 363 : "d" (__func), "a" (__param) : "cc", "memory");
@@ -354,6 +367,47 @@ static inline int crypt_s390_kmctr(long func, void *param, u8 *dest,
354} 367}
355 368
356/** 369/**
370 * crypt_s390_ppno:
371 * @func: the function code passed to PPNO; see crypt_s390_ppno_func
372 * @param: address of parameter block; see POP for details on each func
373 * @dest: address of destination memory area
374 * @dest_len: size of destination memory area in bytes
375 * @seed: address of seed data
376 * @seed_len: size of seed data in bytes
377 *
378 * Executes the PPNO (PERFORM PSEUDORANDOM NUMBER OPERATION)
379 * operation of the CPU.
380 *
381 * Returns -1 for failure, 0 for the query func, number of random
382 * bytes stored in dest buffer for generate function
383 */
384static inline int crypt_s390_ppno(long func, void *param,
385 u8 *dest, long dest_len,
386 const u8 *seed, long seed_len)
387{
388 register long __func asm("0") = func & CRYPT_S390_FUNC_MASK;
389 register void *__param asm("1") = param; /* param block (240 bytes) */
390 register u8 *__dest asm("2") = dest; /* buf for recv random bytes */
391 register long __dest_len asm("3") = dest_len; /* requested random bytes */
392 register const u8 *__seed asm("4") = seed; /* buf with seed data */
393 register long __seed_len asm("5") = seed_len; /* bytes in seed buf */
394 int ret = -1;
395
396 asm volatile (
397 "0: .insn rre,0xb93c0000,%1,%5\n" /* PPNO opcode */
398 "1: brc 1,0b\n" /* handle partial completion */
399 " la %0,0\n"
400 "2:\n"
401 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
402 : "+d" (ret), "+a"(__dest), "+d"(__dest_len)
403 : "d"(__func), "a"(__param), "a"(__seed), "d"(__seed_len)
404 : "cc", "memory");
405 if (ret < 0)
406 return ret;
407 return (func & CRYPT_S390_FUNC_MASK) ? dest_len - __dest_len : 0;
408}
409
410/**
357 * crypt_s390_func_available: 411 * crypt_s390_func_available:
358 * @func: the function code of the specific function; 0 if op in general 412 * @func: the function code of the specific function; 0 if op in general
359 * 413 *
@@ -373,6 +427,9 @@ static inline int crypt_s390_func_available(int func,
373 return 0; 427 return 0;
374 if (facility_mask & CRYPT_S390_MSA4 && !test_facility(77)) 428 if (facility_mask & CRYPT_S390_MSA4 && !test_facility(77))
375 return 0; 429 return 0;
430 if (facility_mask & CRYPT_S390_MSA5 && !test_facility(57))
431 return 0;
432
376 switch (func & CRYPT_S390_OP_MASK) { 433 switch (func & CRYPT_S390_OP_MASK) {
377 case CRYPT_S390_KM: 434 case CRYPT_S390_KM:
378 ret = crypt_s390_km(KM_QUERY, &status, NULL, NULL, 0); 435 ret = crypt_s390_km(KM_QUERY, &status, NULL, NULL, 0);
@@ -390,8 +447,12 @@ static inline int crypt_s390_func_available(int func,
390 ret = crypt_s390_kmac(KMAC_QUERY, &status, NULL, 0); 447 ret = crypt_s390_kmac(KMAC_QUERY, &status, NULL, 0);
391 break; 448 break;
392 case CRYPT_S390_KMCTR: 449 case CRYPT_S390_KMCTR:
393 ret = crypt_s390_kmctr(KMCTR_QUERY, &status, NULL, NULL, 0, 450 ret = crypt_s390_kmctr(KMCTR_QUERY, &status,
394 NULL); 451 NULL, NULL, 0, NULL);
452 break;
453 case CRYPT_S390_PPNO:
454 ret = crypt_s390_ppno(PPNO_QUERY, &status,
455 NULL, 0, NULL, 0);
395 break; 456 break;
396 default: 457 default:
397 return 0; 458 return 0;
@@ -419,15 +480,14 @@ static inline int crypt_s390_pcc(long func, void *param)
419 int ret = -1; 480 int ret = -1;
420 481
421 asm volatile( 482 asm volatile(
422 "0: .insn rre,0xb92c0000,0,0 \n" /* PCC opcode */ 483 "0: .insn rre,0xb92c0000,0,0\n" /* PCC opcode */
423 "1: brc 1,0b \n" /* handle partial completion */ 484 "1: brc 1,0b\n" /* handle partial completion */
424 " la %0,0\n" 485 " la %0,0\n"
425 "2:\n" 486 "2:\n"
426 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 487 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
427 : "+d" (ret) 488 : "+d" (ret)
428 : "d" (__func), "a" (__param) : "cc", "memory"); 489 : "d" (__func), "a" (__param) : "cc", "memory");
429 return ret; 490 return ret;
430} 491}
431 492
432
433#endif /* _CRYPTO_ARCH_S390_CRYPT_S390_H */ 493#endif /* _CRYPTO_ARCH_S390_CRYPT_S390_H */
diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c
index 94a35a4c1b48..1f374b39a4ec 100644
--- a/arch/s390/crypto/prng.c
+++ b/arch/s390/crypto/prng.c
@@ -1,106 +1,529 @@
1/* 1/*
2 * Copyright IBM Corp. 2006, 2007 2 * Copyright IBM Corp. 2006, 2015
3 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 3 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
4 * Harald Freudenberger <freude@de.ibm.com>
4 * Driver for the s390 pseudo random number generator 5 * Driver for the s390 pseudo random number generator
5 */ 6 */
7
8#define KMSG_COMPONENT "prng"
9#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
6#include <linux/fs.h> 11#include <linux/fs.h>
12#include <linux/fips.h>
7#include <linux/init.h> 13#include <linux/init.h>
8#include <linux/kernel.h> 14#include <linux/kernel.h>
15#include <linux/device.h>
9#include <linux/miscdevice.h> 16#include <linux/miscdevice.h>
10#include <linux/module.h> 17#include <linux/module.h>
11#include <linux/moduleparam.h> 18#include <linux/moduleparam.h>
19#include <linux/mutex.h>
12#include <linux/random.h> 20#include <linux/random.h>
13#include <linux/slab.h> 21#include <linux/slab.h>
14#include <asm/debug.h> 22#include <asm/debug.h>
15#include <asm/uaccess.h> 23#include <asm/uaccess.h>
24#include <asm/timex.h>
16 25
17#include "crypt_s390.h" 26#include "crypt_s390.h"
18 27
19MODULE_LICENSE("GPL"); 28MODULE_LICENSE("GPL");
20MODULE_AUTHOR("Jan Glauber <jan.glauber@de.ibm.com>"); 29MODULE_AUTHOR("IBM Corporation");
21MODULE_DESCRIPTION("s390 PRNG interface"); 30MODULE_DESCRIPTION("s390 PRNG interface");
22 31
23static int prng_chunk_size = 256; 32
24module_param(prng_chunk_size, int, S_IRUSR | S_IRGRP | S_IROTH); 33#define PRNG_MODE_AUTO 0
34#define PRNG_MODE_TDES 1
35#define PRNG_MODE_SHA512 2
36
37static unsigned int prng_mode = PRNG_MODE_AUTO;
38module_param_named(mode, prng_mode, int, 0);
39MODULE_PARM_DESC(prng_mode, "PRNG mode: 0 - auto, 1 - TDES, 2 - SHA512");
40
41
42#define PRNG_CHUNKSIZE_TDES_MIN 8
43#define PRNG_CHUNKSIZE_TDES_MAX (64*1024)
44#define PRNG_CHUNKSIZE_SHA512_MIN 64
45#define PRNG_CHUNKSIZE_SHA512_MAX (64*1024)
46
47static unsigned int prng_chunk_size = 256;
48module_param_named(chunksize, prng_chunk_size, int, 0);
25MODULE_PARM_DESC(prng_chunk_size, "PRNG read chunk size in bytes"); 49MODULE_PARM_DESC(prng_chunk_size, "PRNG read chunk size in bytes");
26 50
27static int prng_entropy_limit = 4096; 51
28module_param(prng_entropy_limit, int, S_IRUSR | S_IRGRP | S_IROTH | S_IWUSR); 52#define PRNG_RESEED_LIMIT_TDES 4096
29MODULE_PARM_DESC(prng_entropy_limit, 53#define PRNG_RESEED_LIMIT_TDES_LOWER 4096
30 "PRNG add entropy after that much bytes were produced"); 54#define PRNG_RESEED_LIMIT_SHA512 100000
55#define PRNG_RESEED_LIMIT_SHA512_LOWER 10000
56
57static unsigned int prng_reseed_limit;
58module_param_named(reseed_limit, prng_reseed_limit, int, 0);
59MODULE_PARM_DESC(prng_reseed_limit, "PRNG reseed limit");
60
31 61
32/* 62/*
33 * Any one who considers arithmetical methods of producing random digits is, 63 * Any one who considers arithmetical methods of producing random digits is,
34 * of course, in a state of sin. -- John von Neumann 64 * of course, in a state of sin. -- John von Neumann
35 */ 65 */
36 66
37struct s390_prng_data { 67static int prng_errorflag;
38 unsigned long count; /* how many bytes were produced */ 68
39 char *buf; 69#define PRNG_GEN_ENTROPY_FAILED 1
70#define PRNG_SELFTEST_FAILED 2
71#define PRNG_INSTANTIATE_FAILED 3
72#define PRNG_SEED_FAILED 4
73#define PRNG_RESEED_FAILED 5
74#define PRNG_GEN_FAILED 6
75
76struct prng_ws_s {
77 u8 parm_block[32];
78 u32 reseed_counter;
79 u64 byte_counter;
40}; 80};
41 81
42static struct s390_prng_data *p; 82struct ppno_ws_s {
83 u32 res;
84 u32 reseed_counter;
85 u64 stream_bytes;
86 u8 V[112];
87 u8 C[112];
88};
43 89
44/* copied from libica, use a non-zero initial parameter block */ 90struct prng_data_s {
45static unsigned char parm_block[32] = { 91 struct mutex mutex;
460x0F,0x2B,0x8E,0x63,0x8C,0x8E,0xD2,0x52,0x64,0xB7,0xA0,0x7B,0x75,0x28,0xB8,0xF4, 92 union {
470x75,0x5F,0xD2,0xA6,0x8D,0x97,0x11,0xFF,0x49,0xD8,0x23,0xF3,0x7E,0x21,0xEC,0xA0, 93 struct prng_ws_s prngws;
94 struct ppno_ws_s ppnows;
95 };
96 u8 *buf;
97 u32 rest;
98 u8 *prev;
48}; 99};
49 100
50static int prng_open(struct inode *inode, struct file *file) 101static struct prng_data_s *prng_data;
102
103/* initial parameter block for tdes mode, copied from libica */
104static const u8 initial_parm_block[32] __initconst = {
105 0x0F, 0x2B, 0x8E, 0x63, 0x8C, 0x8E, 0xD2, 0x52,
106 0x64, 0xB7, 0xA0, 0x7B, 0x75, 0x28, 0xB8, 0xF4,
107 0x75, 0x5F, 0xD2, 0xA6, 0x8D, 0x97, 0x11, 0xFF,
108 0x49, 0xD8, 0x23, 0xF3, 0x7E, 0x21, 0xEC, 0xA0 };
109
110
111/*** helper functions ***/
112
113static int generate_entropy(u8 *ebuf, size_t nbytes)
51{ 114{
52 return nonseekable_open(inode, file); 115 int n, ret = 0;
116 u8 *pg, *h, hash[32];
117
118 pg = (u8 *) __get_free_page(GFP_KERNEL);
119 if (!pg) {
120 prng_errorflag = PRNG_GEN_ENTROPY_FAILED;
121 return -ENOMEM;
122 }
123
124 while (nbytes) {
125 /* fill page with urandom bytes */
126 get_random_bytes(pg, PAGE_SIZE);
127 /* exor page with stckf values */
128 for (n = 0; n < sizeof(PAGE_SIZE/sizeof(u64)); n++) {
129 u64 *p = ((u64 *)pg) + n;
130 *p ^= get_tod_clock_fast();
131 }
132 n = (nbytes < sizeof(hash)) ? nbytes : sizeof(hash);
133 if (n < sizeof(hash))
134 h = hash;
135 else
136 h = ebuf;
137 /* generate sha256 from this page */
138 if (crypt_s390_kimd(KIMD_SHA_256, h,
139 pg, PAGE_SIZE) != PAGE_SIZE) {
140 prng_errorflag = PRNG_GEN_ENTROPY_FAILED;
141 ret = -EIO;
142 goto out;
143 }
144 if (n < sizeof(hash))
145 memcpy(ebuf, hash, n);
146 ret += n;
147 ebuf += n;
148 nbytes -= n;
149 }
150
151out:
152 free_page((unsigned long)pg);
153 return ret;
53} 154}
54 155
55static void prng_add_entropy(void) 156
157/*** tdes functions ***/
158
159static void prng_tdes_add_entropy(void)
56{ 160{
57 __u64 entropy[4]; 161 __u64 entropy[4];
58 unsigned int i; 162 unsigned int i;
59 int ret; 163 int ret;
60 164
61 for (i = 0; i < 16; i++) { 165 for (i = 0; i < 16; i++) {
62 ret = crypt_s390_kmc(KMC_PRNG, parm_block, (char *)entropy, 166 ret = crypt_s390_kmc(KMC_PRNG, prng_data->prngws.parm_block,
63 (char *)entropy, sizeof(entropy)); 167 (char *)entropy, (char *)entropy,
168 sizeof(entropy));
64 BUG_ON(ret < 0 || ret != sizeof(entropy)); 169 BUG_ON(ret < 0 || ret != sizeof(entropy));
65 memcpy(parm_block, entropy, sizeof(entropy)); 170 memcpy(prng_data->prngws.parm_block, entropy, sizeof(entropy));
66 } 171 }
67} 172}
68 173
69static void prng_seed(int nbytes) 174
175static void prng_tdes_seed(int nbytes)
70{ 176{
71 char buf[16]; 177 char buf[16];
72 int i = 0; 178 int i = 0;
73 179
74 BUG_ON(nbytes > 16); 180 BUG_ON(nbytes > sizeof(buf));
181
75 get_random_bytes(buf, nbytes); 182 get_random_bytes(buf, nbytes);
76 183
77 /* Add the entropy */ 184 /* Add the entropy */
78 while (nbytes >= 8) { 185 while (nbytes >= 8) {
79 *((__u64 *)parm_block) ^= *((__u64 *)(buf+i)); 186 *((__u64 *)prng_data->prngws.parm_block) ^= *((__u64 *)(buf+i));
80 prng_add_entropy(); 187 prng_tdes_add_entropy();
81 i += 8; 188 i += 8;
82 nbytes -= 8; 189 nbytes -= 8;
83 } 190 }
84 prng_add_entropy(); 191 prng_tdes_add_entropy();
192 prng_data->prngws.reseed_counter = 0;
193}
194
195
196static int __init prng_tdes_instantiate(void)
197{
198 int datalen;
199
200 pr_debug("prng runs in TDES mode with "
201 "chunksize=%d and reseed_limit=%u\n",
202 prng_chunk_size, prng_reseed_limit);
203
204 /* memory allocation, prng_data struct init, mutex init */
205 datalen = sizeof(struct prng_data_s) + prng_chunk_size;
206 prng_data = kzalloc(datalen, GFP_KERNEL);
207 if (!prng_data) {
208 prng_errorflag = PRNG_INSTANTIATE_FAILED;
209 return -ENOMEM;
210 }
211 mutex_init(&prng_data->mutex);
212 prng_data->buf = ((u8 *)prng_data) + sizeof(struct prng_data_s);
213 memcpy(prng_data->prngws.parm_block, initial_parm_block, 32);
214
215 /* initialize the PRNG, add 128 bits of entropy */
216 prng_tdes_seed(16);
217
218 return 0;
85} 219}
86 220
87static ssize_t prng_read(struct file *file, char __user *ubuf, size_t nbytes, 221
88 loff_t *ppos) 222static void prng_tdes_deinstantiate(void)
223{
224 pr_debug("The prng module stopped "
225 "after running in triple DES mode\n");
226 kzfree(prng_data);
227}
228
229
230/*** sha512 functions ***/
231
232static int __init prng_sha512_selftest(void)
89{ 233{
90 int chunk, n; 234 /* NIST DRBG testvector for Hash Drbg, Sha-512, Count #0 */
235 static const u8 seed[] __initconst = {
236 0x6b, 0x50, 0xa7, 0xd8, 0xf8, 0xa5, 0x5d, 0x7a,
237 0x3d, 0xf8, 0xbb, 0x40, 0xbc, 0xc3, 0xb7, 0x22,
238 0xd8, 0x70, 0x8d, 0xe6, 0x7f, 0xda, 0x01, 0x0b,
239 0x03, 0xc4, 0xc8, 0x4d, 0x72, 0x09, 0x6f, 0x8c,
240 0x3e, 0xc6, 0x49, 0xcc, 0x62, 0x56, 0xd9, 0xfa,
241 0x31, 0xdb, 0x7a, 0x29, 0x04, 0xaa, 0xf0, 0x25 };
242 static const u8 V0[] __initconst = {
243 0x00, 0xad, 0xe3, 0x6f, 0x9a, 0x01, 0xc7, 0x76,
244 0x61, 0x34, 0x35, 0xf5, 0x4e, 0x24, 0x74, 0x22,
245 0x21, 0x9a, 0x29, 0x89, 0xc7, 0x93, 0x2e, 0x60,
246 0x1e, 0xe8, 0x14, 0x24, 0x8d, 0xd5, 0x03, 0xf1,
247 0x65, 0x5d, 0x08, 0x22, 0x72, 0xd5, 0xad, 0x95,
248 0xe1, 0x23, 0x1e, 0x8a, 0xa7, 0x13, 0xd9, 0x2b,
249 0x5e, 0xbc, 0xbb, 0x80, 0xab, 0x8d, 0xe5, 0x79,
250 0xab, 0x5b, 0x47, 0x4e, 0xdd, 0xee, 0x6b, 0x03,
251 0x8f, 0x0f, 0x5c, 0x5e, 0xa9, 0x1a, 0x83, 0xdd,
252 0xd3, 0x88, 0xb2, 0x75, 0x4b, 0xce, 0x83, 0x36,
253 0x57, 0x4b, 0xf1, 0x5c, 0xca, 0x7e, 0x09, 0xc0,
254 0xd3, 0x89, 0xc6, 0xe0, 0xda, 0xc4, 0x81, 0x7e,
255 0x5b, 0xf9, 0xe1, 0x01, 0xc1, 0x92, 0x05, 0xea,
256 0xf5, 0x2f, 0xc6, 0xc6, 0xc7, 0x8f, 0xbc, 0xf4 };
257 static const u8 C0[] __initconst = {
258 0x00, 0xf4, 0xa3, 0xe5, 0xa0, 0x72, 0x63, 0x95,
259 0xc6, 0x4f, 0x48, 0xd0, 0x8b, 0x5b, 0x5f, 0x8e,
260 0x6b, 0x96, 0x1f, 0x16, 0xed, 0xbc, 0x66, 0x94,
261 0x45, 0x31, 0xd7, 0x47, 0x73, 0x22, 0xa5, 0x86,
262 0xce, 0xc0, 0x4c, 0xac, 0x63, 0xb8, 0x39, 0x50,
263 0xbf, 0xe6, 0x59, 0x6c, 0x38, 0x58, 0x99, 0x1f,
264 0x27, 0xa7, 0x9d, 0x71, 0x2a, 0xb3, 0x7b, 0xf9,
265 0xfb, 0x17, 0x86, 0xaa, 0x99, 0x81, 0xaa, 0x43,
266 0xe4, 0x37, 0xd3, 0x1e, 0x6e, 0xe5, 0xe6, 0xee,
267 0xc2, 0xed, 0x95, 0x4f, 0x53, 0x0e, 0x46, 0x8a,
268 0xcc, 0x45, 0xa5, 0xdb, 0x69, 0x0d, 0x81, 0xc9,
269 0x32, 0x92, 0xbc, 0x8f, 0x33, 0xe6, 0xf6, 0x09,
270 0x7c, 0x8e, 0x05, 0x19, 0x0d, 0xf1, 0xb6, 0xcc,
271 0xf3, 0x02, 0x21, 0x90, 0x25, 0xec, 0xed, 0x0e };
272 static const u8 random[] __initconst = {
273 0x95, 0xb7, 0xf1, 0x7e, 0x98, 0x02, 0xd3, 0x57,
274 0x73, 0x92, 0xc6, 0xa9, 0xc0, 0x80, 0x83, 0xb6,
275 0x7d, 0xd1, 0x29, 0x22, 0x65, 0xb5, 0xf4, 0x2d,
276 0x23, 0x7f, 0x1c, 0x55, 0xbb, 0x9b, 0x10, 0xbf,
277 0xcf, 0xd8, 0x2c, 0x77, 0xa3, 0x78, 0xb8, 0x26,
278 0x6a, 0x00, 0x99, 0x14, 0x3b, 0x3c, 0x2d, 0x64,
279 0x61, 0x1e, 0xee, 0xb6, 0x9a, 0xcd, 0xc0, 0x55,
280 0x95, 0x7c, 0x13, 0x9e, 0x8b, 0x19, 0x0c, 0x7a,
281 0x06, 0x95, 0x5f, 0x2c, 0x79, 0x7c, 0x27, 0x78,
282 0xde, 0x94, 0x03, 0x96, 0xa5, 0x01, 0xf4, 0x0e,
283 0x91, 0x39, 0x6a, 0xcf, 0x8d, 0x7e, 0x45, 0xeb,
284 0xdb, 0xb5, 0x3b, 0xbf, 0x8c, 0x97, 0x52, 0x30,
285 0xd2, 0xf0, 0xff, 0x91, 0x06, 0xc7, 0x61, 0x19,
286 0xae, 0x49, 0x8e, 0x7f, 0xbc, 0x03, 0xd9, 0x0f,
287 0x8e, 0x4c, 0x51, 0x62, 0x7a, 0xed, 0x5c, 0x8d,
288 0x42, 0x63, 0xd5, 0xd2, 0xb9, 0x78, 0x87, 0x3a,
289 0x0d, 0xe5, 0x96, 0xee, 0x6d, 0xc7, 0xf7, 0xc2,
290 0x9e, 0x37, 0xee, 0xe8, 0xb3, 0x4c, 0x90, 0xdd,
291 0x1c, 0xf6, 0xa9, 0xdd, 0xb2, 0x2b, 0x4c, 0xbd,
292 0x08, 0x6b, 0x14, 0xb3, 0x5d, 0xe9, 0x3d, 0xa2,
293 0xd5, 0xcb, 0x18, 0x06, 0x69, 0x8c, 0xbd, 0x7b,
294 0xbb, 0x67, 0xbf, 0xe3, 0xd3, 0x1f, 0xd2, 0xd1,
295 0xdb, 0xd2, 0xa1, 0xe0, 0x58, 0xa3, 0xeb, 0x99,
296 0xd7, 0xe5, 0x1f, 0x1a, 0x93, 0x8e, 0xed, 0x5e,
297 0x1c, 0x1d, 0xe2, 0x3a, 0x6b, 0x43, 0x45, 0xd3,
298 0x19, 0x14, 0x09, 0xf9, 0x2f, 0x39, 0xb3, 0x67,
299 0x0d, 0x8d, 0xbf, 0xb6, 0x35, 0xd8, 0xe6, 0xa3,
300 0x69, 0x32, 0xd8, 0x10, 0x33, 0xd1, 0x44, 0x8d,
301 0x63, 0xb4, 0x03, 0xdd, 0xf8, 0x8e, 0x12, 0x1b,
302 0x6e, 0x81, 0x9a, 0xc3, 0x81, 0x22, 0x6c, 0x13,
303 0x21, 0xe4, 0xb0, 0x86, 0x44, 0xf6, 0x72, 0x7c,
304 0x36, 0x8c, 0x5a, 0x9f, 0x7a, 0x4b, 0x3e, 0xe2 };
305
91 int ret = 0; 306 int ret = 0;
92 int tmp; 307 u8 buf[sizeof(random)];
308 struct ppno_ws_s ws;
309
310 memset(&ws, 0, sizeof(ws));
311
312 /* initial seed */
313 ret = crypt_s390_ppno(PPNO_SHA512_DRNG_SEED,
314 &ws, NULL, 0,
315 seed, sizeof(seed));
316 if (ret < 0) {
317 pr_err("The prng self test seed operation for the "
318 "SHA-512 mode failed with rc=%d\n", ret);
319 prng_errorflag = PRNG_SELFTEST_FAILED;
320 return -EIO;
321 }
322
323 /* check working states V and C */
324 if (memcmp(ws.V, V0, sizeof(V0)) != 0
325 || memcmp(ws.C, C0, sizeof(C0)) != 0) {
326 pr_err("The prng self test state test "
327 "for the SHA-512 mode failed\n");
328 prng_errorflag = PRNG_SELFTEST_FAILED;
329 return -EIO;
330 }
331
332 /* generate random bytes */
333 ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN,
334 &ws, buf, sizeof(buf),
335 NULL, 0);
336 if (ret < 0) {
337 pr_err("The prng self test generate operation for "
338 "the SHA-512 mode failed with rc=%d\n", ret);
339 prng_errorflag = PRNG_SELFTEST_FAILED;
340 return -EIO;
341 }
342 ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN,
343 &ws, buf, sizeof(buf),
344 NULL, 0);
345 if (ret < 0) {
346 pr_err("The prng self test generate operation for "
347 "the SHA-512 mode failed with rc=%d\n", ret);
348 prng_errorflag = PRNG_SELFTEST_FAILED;
349 return -EIO;
350 }
351
352 /* check against expected data */
353 if (memcmp(buf, random, sizeof(random)) != 0) {
354 pr_err("The prng self test data test "
355 "for the SHA-512 mode failed\n");
356 prng_errorflag = PRNG_SELFTEST_FAILED;
357 return -EIO;
358 }
359
360 return 0;
361}
362
363
364static int __init prng_sha512_instantiate(void)
365{
366 int ret, datalen;
367 u8 seed[64];
368
369 pr_debug("prng runs in SHA-512 mode "
370 "with chunksize=%d and reseed_limit=%u\n",
371 prng_chunk_size, prng_reseed_limit);
372
373 /* memory allocation, prng_data struct init, mutex init */
374 datalen = sizeof(struct prng_data_s) + prng_chunk_size;
375 if (fips_enabled)
376 datalen += prng_chunk_size;
377 prng_data = kzalloc(datalen, GFP_KERNEL);
378 if (!prng_data) {
379 prng_errorflag = PRNG_INSTANTIATE_FAILED;
380 return -ENOMEM;
381 }
382 mutex_init(&prng_data->mutex);
383 prng_data->buf = ((u8 *)prng_data) + sizeof(struct prng_data_s);
384
385 /* selftest */
386 ret = prng_sha512_selftest();
387 if (ret)
388 goto outfree;
389
390 /* generate initial seed bytestring, first 48 bytes of entropy */
391 ret = generate_entropy(seed, 48);
392 if (ret != 48)
393 goto outfree;
394 /* followed by 16 bytes of unique nonce */
395 get_tod_clock_ext(seed + 48);
396
397 /* initial seed of the ppno drng */
398 ret = crypt_s390_ppno(PPNO_SHA512_DRNG_SEED,
399 &prng_data->ppnows, NULL, 0,
400 seed, sizeof(seed));
401 if (ret < 0) {
402 prng_errorflag = PRNG_SEED_FAILED;
403 ret = -EIO;
404 goto outfree;
405 }
406
407 /* if fips mode is enabled, generate a first block of random
408 bytes for the FIPS 140-2 Conditional Self Test */
409 if (fips_enabled) {
410 prng_data->prev = prng_data->buf + prng_chunk_size;
411 ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN,
412 &prng_data->ppnows,
413 prng_data->prev,
414 prng_chunk_size,
415 NULL, 0);
416 if (ret < 0 || ret != prng_chunk_size) {
417 prng_errorflag = PRNG_GEN_FAILED;
418 ret = -EIO;
419 goto outfree;
420 }
421 }
422
423 return 0;
424
425outfree:
426 kfree(prng_data);
427 return ret;
428}
429
430
431static void prng_sha512_deinstantiate(void)
432{
433 pr_debug("The prng module stopped after running in SHA-512 mode\n");
434 kzfree(prng_data);
435}
436
437
438static int prng_sha512_reseed(void)
439{
440 int ret;
441 u8 seed[32];
442
443 /* generate 32 bytes of fresh entropy */
444 ret = generate_entropy(seed, sizeof(seed));
445 if (ret != sizeof(seed))
446 return ret;
447
448 /* do a reseed of the ppno drng with this bytestring */
449 ret = crypt_s390_ppno(PPNO_SHA512_DRNG_SEED,
450 &prng_data->ppnows, NULL, 0,
451 seed, sizeof(seed));
452 if (ret) {
453 prng_errorflag = PRNG_RESEED_FAILED;
454 return -EIO;
455 }
456
457 return 0;
458}
459
460
461static int prng_sha512_generate(u8 *buf, size_t nbytes)
462{
463 int ret;
464
465 /* reseed needed ? */
466 if (prng_data->ppnows.reseed_counter > prng_reseed_limit) {
467 ret = prng_sha512_reseed();
468 if (ret)
469 return ret;
470 }
471
472 /* PPNO generate */
473 ret = crypt_s390_ppno(PPNO_SHA512_DRNG_GEN,
474 &prng_data->ppnows, buf, nbytes,
475 NULL, 0);
476 if (ret < 0 || ret != nbytes) {
477 prng_errorflag = PRNG_GEN_FAILED;
478 return -EIO;
479 }
480
481 /* FIPS 140-2 Conditional Self Test */
482 if (fips_enabled) {
483 if (!memcmp(prng_data->prev, buf, nbytes)) {
484 prng_errorflag = PRNG_GEN_FAILED;
485 return -EILSEQ;
486 }
487 memcpy(prng_data->prev, buf, nbytes);
488 }
489
490 return ret;
491}
492
493
494/*** file io functions ***/
495
496static int prng_open(struct inode *inode, struct file *file)
497{
498 return nonseekable_open(inode, file);
499}
500
501
502static ssize_t prng_tdes_read(struct file *file, char __user *ubuf,
503 size_t nbytes, loff_t *ppos)
504{
505 int chunk, n, tmp, ret = 0;
506
507 /* lock prng_data struct */
508 if (mutex_lock_interruptible(&prng_data->mutex))
509 return -ERESTARTSYS;
93 510
94 /* nbytes can be arbitrary length, we split it into chunks */
95 while (nbytes) { 511 while (nbytes) {
96 /* same as in extract_entropy_user in random.c */
97 if (need_resched()) { 512 if (need_resched()) {
98 if (signal_pending(current)) { 513 if (signal_pending(current)) {
99 if (ret == 0) 514 if (ret == 0)
100 ret = -ERESTARTSYS; 515 ret = -ERESTARTSYS;
101 break; 516 break;
102 } 517 }
518 /* give mutex free before calling schedule() */
519 mutex_unlock(&prng_data->mutex);
103 schedule(); 520 schedule();
521 /* occopy mutex again */
522 if (mutex_lock_interruptible(&prng_data->mutex)) {
523 if (ret == 0)
524 ret = -ERESTARTSYS;
525 return ret;
526 }
104 } 527 }
105 528
106 /* 529 /*
@@ -112,12 +535,11 @@ static ssize_t prng_read(struct file *file, char __user *ubuf, size_t nbytes,
112 /* PRNG only likes multiples of 8 bytes */ 535 /* PRNG only likes multiples of 8 bytes */
113 n = (chunk + 7) & -8; 536 n = (chunk + 7) & -8;
114 537
115 if (p->count > prng_entropy_limit) 538 if (prng_data->prngws.reseed_counter > prng_reseed_limit)
116 prng_seed(8); 539 prng_tdes_seed(8);
117 540
118 /* if the CPU supports PRNG stckf is present too */ 541 /* if the CPU supports PRNG stckf is present too */
119 asm volatile(".insn s,0xb27c0000,%0" 542 *((unsigned long long *)prng_data->buf) = get_tod_clock_fast();
120 : "=m" (*((unsigned long long *)p->buf)) : : "cc");
121 543
122 /* 544 /*
123 * Beside the STCKF the input for the TDES-EDE is the output 545 * Beside the STCKF the input for the TDES-EDE is the output
@@ -132,35 +554,259 @@ static ssize_t prng_read(struct file *file, char __user *ubuf, size_t nbytes,
132 * Note: you can still get strict X9.17 conformity by setting 554 * Note: you can still get strict X9.17 conformity by setting
133 * prng_chunk_size to 8 bytes. 555 * prng_chunk_size to 8 bytes.
134 */ 556 */
135 tmp = crypt_s390_kmc(KMC_PRNG, parm_block, p->buf, p->buf, n); 557 tmp = crypt_s390_kmc(KMC_PRNG, prng_data->prngws.parm_block,
136 BUG_ON((tmp < 0) || (tmp != n)); 558 prng_data->buf, prng_data->buf, n);
559 if (tmp < 0 || tmp != n) {
560 ret = -EIO;
561 break;
562 }
137 563
138 p->count += n; 564 prng_data->prngws.byte_counter += n;
565 prng_data->prngws.reseed_counter += n;
139 566
140 if (copy_to_user(ubuf, p->buf, chunk)) 567 if (copy_to_user(ubuf, prng_data->buf, chunk))
141 return -EFAULT; 568 return -EFAULT;
142 569
143 nbytes -= chunk; 570 nbytes -= chunk;
144 ret += chunk; 571 ret += chunk;
145 ubuf += chunk; 572 ubuf += chunk;
146 } 573 }
574
575 /* unlock prng_data struct */
576 mutex_unlock(&prng_data->mutex);
577
147 return ret; 578 return ret;
148} 579}
149 580
150static const struct file_operations prng_fops = { 581
582static ssize_t prng_sha512_read(struct file *file, char __user *ubuf,
583 size_t nbytes, loff_t *ppos)
584{
585 int n, ret = 0;
586 u8 *p;
587
588 /* if errorflag is set do nothing and return 'broken pipe' */
589 if (prng_errorflag)
590 return -EPIPE;
591
592 /* lock prng_data struct */
593 if (mutex_lock_interruptible(&prng_data->mutex))
594 return -ERESTARTSYS;
595
596 while (nbytes) {
597 if (need_resched()) {
598 if (signal_pending(current)) {
599 if (ret == 0)
600 ret = -ERESTARTSYS;
601 break;
602 }
603 /* give mutex free before calling schedule() */
604 mutex_unlock(&prng_data->mutex);
605 schedule();
606 /* occopy mutex again */
607 if (mutex_lock_interruptible(&prng_data->mutex)) {
608 if (ret == 0)
609 ret = -ERESTARTSYS;
610 return ret;
611 }
612 }
613 if (prng_data->rest) {
614 /* push left over random bytes from the previous read */
615 p = prng_data->buf + prng_chunk_size - prng_data->rest;
616 n = (nbytes < prng_data->rest) ?
617 nbytes : prng_data->rest;
618 prng_data->rest -= n;
619 } else {
620 /* generate one chunk of random bytes into read buf */
621 p = prng_data->buf;
622 n = prng_sha512_generate(p, prng_chunk_size);
623 if (n < 0) {
624 ret = n;
625 break;
626 }
627 if (nbytes < prng_chunk_size) {
628 n = nbytes;
629 prng_data->rest = prng_chunk_size - n;
630 } else {
631 n = prng_chunk_size;
632 prng_data->rest = 0;
633 }
634 }
635 if (copy_to_user(ubuf, p, n)) {
636 ret = -EFAULT;
637 break;
638 }
639 ubuf += n;
640 nbytes -= n;
641 ret += n;
642 }
643
644 /* unlock prng_data struct */
645 mutex_unlock(&prng_data->mutex);
646
647 return ret;
648}
649
650
651/*** sysfs stuff ***/
652
653static const struct file_operations prng_sha512_fops = {
654 .owner = THIS_MODULE,
655 .open = &prng_open,
656 .release = NULL,
657 .read = &prng_sha512_read,
658 .llseek = noop_llseek,
659};
660static const struct file_operations prng_tdes_fops = {
151 .owner = THIS_MODULE, 661 .owner = THIS_MODULE,
152 .open = &prng_open, 662 .open = &prng_open,
153 .release = NULL, 663 .release = NULL,
154 .read = &prng_read, 664 .read = &prng_tdes_read,
155 .llseek = noop_llseek, 665 .llseek = noop_llseek,
156}; 666};
157 667
158static struct miscdevice prng_dev = { 668static struct miscdevice prng_sha512_dev = {
669 .name = "prandom",
670 .minor = MISC_DYNAMIC_MINOR,
671 .fops = &prng_sha512_fops,
672};
673static struct miscdevice prng_tdes_dev = {
159 .name = "prandom", 674 .name = "prandom",
160 .minor = MISC_DYNAMIC_MINOR, 675 .minor = MISC_DYNAMIC_MINOR,
161 .fops = &prng_fops, 676 .fops = &prng_tdes_fops,
162}; 677};
163 678
679
680/* chunksize attribute (ro) */
681static ssize_t prng_chunksize_show(struct device *dev,
682 struct device_attribute *attr,
683 char *buf)
684{
685 return snprintf(buf, PAGE_SIZE, "%u\n", prng_chunk_size);
686}
687static DEVICE_ATTR(chunksize, 0444, prng_chunksize_show, NULL);
688
689/* counter attribute (ro) */
690static ssize_t prng_counter_show(struct device *dev,
691 struct device_attribute *attr,
692 char *buf)
693{
694 u64 counter;
695
696 if (mutex_lock_interruptible(&prng_data->mutex))
697 return -ERESTARTSYS;
698 if (prng_mode == PRNG_MODE_SHA512)
699 counter = prng_data->ppnows.stream_bytes;
700 else
701 counter = prng_data->prngws.byte_counter;
702 mutex_unlock(&prng_data->mutex);
703
704 return snprintf(buf, PAGE_SIZE, "%llu\n", counter);
705}
706static DEVICE_ATTR(byte_counter, 0444, prng_counter_show, NULL);
707
708/* errorflag attribute (ro) */
709static ssize_t prng_errorflag_show(struct device *dev,
710 struct device_attribute *attr,
711 char *buf)
712{
713 return snprintf(buf, PAGE_SIZE, "%d\n", prng_errorflag);
714}
715static DEVICE_ATTR(errorflag, 0444, prng_errorflag_show, NULL);
716
717/* mode attribute (ro) */
718static ssize_t prng_mode_show(struct device *dev,
719 struct device_attribute *attr,
720 char *buf)
721{
722 if (prng_mode == PRNG_MODE_TDES)
723 return snprintf(buf, PAGE_SIZE, "TDES\n");
724 else
725 return snprintf(buf, PAGE_SIZE, "SHA512\n");
726}
727static DEVICE_ATTR(mode, 0444, prng_mode_show, NULL);
728
729/* reseed attribute (w) */
730static ssize_t prng_reseed_store(struct device *dev,
731 struct device_attribute *attr,
732 const char *buf, size_t count)
733{
734 if (mutex_lock_interruptible(&prng_data->mutex))
735 return -ERESTARTSYS;
736 prng_sha512_reseed();
737 mutex_unlock(&prng_data->mutex);
738
739 return count;
740}
741static DEVICE_ATTR(reseed, 0200, NULL, prng_reseed_store);
742
743/* reseed limit attribute (rw) */
744static ssize_t prng_reseed_limit_show(struct device *dev,
745 struct device_attribute *attr,
746 char *buf)
747{
748 return snprintf(buf, PAGE_SIZE, "%u\n", prng_reseed_limit);
749}
750static ssize_t prng_reseed_limit_store(struct device *dev,
751 struct device_attribute *attr,
752 const char *buf, size_t count)
753{
754 unsigned limit;
755
756 if (sscanf(buf, "%u\n", &limit) != 1)
757 return -EINVAL;
758
759 if (prng_mode == PRNG_MODE_SHA512) {
760 if (limit < PRNG_RESEED_LIMIT_SHA512_LOWER)
761 return -EINVAL;
762 } else {
763 if (limit < PRNG_RESEED_LIMIT_TDES_LOWER)
764 return -EINVAL;
765 }
766
767 prng_reseed_limit = limit;
768
769 return count;
770}
771static DEVICE_ATTR(reseed_limit, 0644,
772 prng_reseed_limit_show, prng_reseed_limit_store);
773
774/* strength attribute (ro) */
775static ssize_t prng_strength_show(struct device *dev,
776 struct device_attribute *attr,
777 char *buf)
778{
779 return snprintf(buf, PAGE_SIZE, "256\n");
780}
781static DEVICE_ATTR(strength, 0444, prng_strength_show, NULL);
782
783static struct attribute *prng_sha512_dev_attrs[] = {
784 &dev_attr_errorflag.attr,
785 &dev_attr_chunksize.attr,
786 &dev_attr_byte_counter.attr,
787 &dev_attr_mode.attr,
788 &dev_attr_reseed.attr,
789 &dev_attr_reseed_limit.attr,
790 &dev_attr_strength.attr,
791 NULL
792};
793static struct attribute *prng_tdes_dev_attrs[] = {
794 &dev_attr_chunksize.attr,
795 &dev_attr_byte_counter.attr,
796 &dev_attr_mode.attr,
797 NULL
798};
799
800static struct attribute_group prng_sha512_dev_attr_group = {
801 .attrs = prng_sha512_dev_attrs
802};
803static struct attribute_group prng_tdes_dev_attr_group = {
804 .attrs = prng_tdes_dev_attrs
805};
806
807
808/*** module init and exit ***/
809
164static int __init prng_init(void) 810static int __init prng_init(void)
165{ 811{
166 int ret; 812 int ret;
@@ -169,43 +815,105 @@ static int __init prng_init(void)
169 if (!crypt_s390_func_available(KMC_PRNG, CRYPT_S390_MSA)) 815 if (!crypt_s390_func_available(KMC_PRNG, CRYPT_S390_MSA))
170 return -EOPNOTSUPP; 816 return -EOPNOTSUPP;
171 817
172 if (prng_chunk_size < 8) 818 /* choose prng mode */
173 return -EINVAL; 819 if (prng_mode != PRNG_MODE_TDES) {
820 /* check for MSA5 support for PPNO operations */
821 if (!crypt_s390_func_available(PPNO_SHA512_DRNG_GEN,
822 CRYPT_S390_MSA5)) {
823 if (prng_mode == PRNG_MODE_SHA512) {
824 pr_err("The prng module cannot "
825 "start in SHA-512 mode\n");
826 return -EOPNOTSUPP;
827 }
828 prng_mode = PRNG_MODE_TDES;
829 } else
830 prng_mode = PRNG_MODE_SHA512;
831 }
174 832
175 p = kmalloc(sizeof(struct s390_prng_data), GFP_KERNEL); 833 if (prng_mode == PRNG_MODE_SHA512) {
176 if (!p)
177 return -ENOMEM;
178 p->count = 0;
179 834
180 p->buf = kmalloc(prng_chunk_size, GFP_KERNEL); 835 /* SHA512 mode */
181 if (!p->buf) {
182 ret = -ENOMEM;
183 goto out_free;
184 }
185 836
186 /* initialize the PRNG, add 128 bits of entropy */ 837 if (prng_chunk_size < PRNG_CHUNKSIZE_SHA512_MIN
187 prng_seed(16); 838 || prng_chunk_size > PRNG_CHUNKSIZE_SHA512_MAX)
839 return -EINVAL;
840 prng_chunk_size = (prng_chunk_size + 0x3f) & ~0x3f;
188 841
189 ret = misc_register(&prng_dev); 842 if (prng_reseed_limit == 0)
190 if (ret) 843 prng_reseed_limit = PRNG_RESEED_LIMIT_SHA512;
191 goto out_buf; 844 else if (prng_reseed_limit < PRNG_RESEED_LIMIT_SHA512_LOWER)
192 return 0; 845 return -EINVAL;
846
847 ret = prng_sha512_instantiate();
848 if (ret)
849 goto out;
850
851 ret = misc_register(&prng_sha512_dev);
852 if (ret) {
853 prng_sha512_deinstantiate();
854 goto out;
855 }
856 ret = sysfs_create_group(&prng_sha512_dev.this_device->kobj,
857 &prng_sha512_dev_attr_group);
858 if (ret) {
859 misc_deregister(&prng_sha512_dev);
860 prng_sha512_deinstantiate();
861 goto out;
862 }
193 863
194out_buf: 864 } else {
195 kfree(p->buf); 865
196out_free: 866 /* TDES mode */
197 kfree(p); 867
868 if (prng_chunk_size < PRNG_CHUNKSIZE_TDES_MIN
869 || prng_chunk_size > PRNG_CHUNKSIZE_TDES_MAX)
870 return -EINVAL;
871 prng_chunk_size = (prng_chunk_size + 0x07) & ~0x07;
872
873 if (prng_reseed_limit == 0)
874 prng_reseed_limit = PRNG_RESEED_LIMIT_TDES;
875 else if (prng_reseed_limit < PRNG_RESEED_LIMIT_TDES_LOWER)
876 return -EINVAL;
877
878 ret = prng_tdes_instantiate();
879 if (ret)
880 goto out;
881
882 ret = misc_register(&prng_tdes_dev);
883 if (ret) {
884 prng_tdes_deinstantiate();
885 goto out;
886 }
887 ret = sysfs_create_group(&prng_tdes_dev.this_device->kobj,
888 &prng_tdes_dev_attr_group);
889 if (ret) {
890 misc_deregister(&prng_tdes_dev);
891 prng_tdes_deinstantiate();
892 goto out;
893 }
894
895 }
896
897out:
198 return ret; 898 return ret;
199} 899}
200 900
901
201static void __exit prng_exit(void) 902static void __exit prng_exit(void)
202{ 903{
203 /* wipe me */ 904 if (prng_mode == PRNG_MODE_SHA512) {
204 kzfree(p->buf); 905 sysfs_remove_group(&prng_sha512_dev.this_device->kobj,
205 kfree(p); 906 &prng_sha512_dev_attr_group);
206 907 misc_deregister(&prng_sha512_dev);
207 misc_deregister(&prng_dev); 908 prng_sha512_deinstantiate();
909 } else {
910 sysfs_remove_group(&prng_tdes_dev.this_device->kobj,
911 &prng_tdes_dev_attr_group);
912 misc_deregister(&prng_tdes_dev);
913 prng_tdes_deinstantiate();
914 }
208} 915}
209 916
917
210module_init(prng_init); 918module_init(prng_init);
211module_exit(prng_exit); 919module_exit(prng_exit);
diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h
index 694bcd6bd927..2f924bc30e35 100644
--- a/arch/s390/include/asm/kexec.h
+++ b/arch/s390/include/asm/kexec.h
@@ -26,6 +26,9 @@
26/* Not more than 2GB */ 26/* Not more than 2GB */
27#define KEXEC_CONTROL_MEMORY_LIMIT (1UL<<31) 27#define KEXEC_CONTROL_MEMORY_LIMIT (1UL<<31)
28 28
29/* Allocate control page with GFP_DMA */
30#define KEXEC_CONTROL_MEMORY_GFP GFP_DMA
31
29/* Maximum address we can use for the crash control pages */ 32/* Maximum address we can use for the crash control pages */
30#define KEXEC_CRASH_CONTROL_MEMORY_LIMIT (-1UL) 33#define KEXEC_CRASH_CONTROL_MEMORY_LIMIT (-1UL)
31 34
diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h
index a5e656260a70..d29ad9545b41 100644
--- a/arch/s390/include/asm/mmu.h
+++ b/arch/s390/include/asm/mmu.h
@@ -14,7 +14,9 @@ typedef struct {
14 unsigned long asce_bits; 14 unsigned long asce_bits;
15 unsigned long asce_limit; 15 unsigned long asce_limit;
16 unsigned long vdso_base; 16 unsigned long vdso_base;
17 /* The mmu context has extended page tables. */ 17 /* The mmu context allocates 4K page tables. */
18 unsigned int alloc_pgste:1;
19 /* The mmu context uses extended page tables. */
18 unsigned int has_pgste:1; 20 unsigned int has_pgste:1;
19 /* The mmu context uses storage keys. */ 21 /* The mmu context uses storage keys. */
20 unsigned int use_skey:1; 22 unsigned int use_skey:1;
diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index d25d9ff10ba8..fb1b93ea3e3f 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -20,8 +20,11 @@ static inline int init_new_context(struct task_struct *tsk,
20 mm->context.flush_mm = 0; 20 mm->context.flush_mm = 0;
21 mm->context.asce_bits = _ASCE_TABLE_LENGTH | _ASCE_USER_BITS; 21 mm->context.asce_bits = _ASCE_TABLE_LENGTH | _ASCE_USER_BITS;
22 mm->context.asce_bits |= _ASCE_TYPE_REGION3; 22 mm->context.asce_bits |= _ASCE_TYPE_REGION3;
23#ifdef CONFIG_PGSTE
24 mm->context.alloc_pgste = page_table_allocate_pgste;
23 mm->context.has_pgste = 0; 25 mm->context.has_pgste = 0;
24 mm->context.use_skey = 0; 26 mm->context.use_skey = 0;
27#endif
25 mm->context.asce_limit = STACK_TOP_MAX; 28 mm->context.asce_limit = STACK_TOP_MAX;
26 crst_table_init((unsigned long *) mm->pgd, pgd_entry_type(mm)); 29 crst_table_init((unsigned long *) mm->pgd, pgd_entry_type(mm));
27 return 0; 30 return 0;
diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h
index 51e7fb634ebc..7b7858f158b4 100644
--- a/arch/s390/include/asm/pgalloc.h
+++ b/arch/s390/include/asm/pgalloc.h
@@ -21,6 +21,7 @@ void crst_table_free(struct mm_struct *, unsigned long *);
21unsigned long *page_table_alloc(struct mm_struct *); 21unsigned long *page_table_alloc(struct mm_struct *);
22void page_table_free(struct mm_struct *, unsigned long *); 22void page_table_free(struct mm_struct *, unsigned long *);
23void page_table_free_rcu(struct mmu_gather *, unsigned long *, unsigned long); 23void page_table_free_rcu(struct mmu_gather *, unsigned long *, unsigned long);
24extern int page_table_allocate_pgste;
24 25
25int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, 26int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
26 unsigned long key, bool nq); 27 unsigned long key, bool nq);
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 989cfae9e202..fc642399b489 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -12,12 +12,9 @@
12#define _ASM_S390_PGTABLE_H 12#define _ASM_S390_PGTABLE_H
13 13
14/* 14/*
15 * The Linux memory management assumes a three-level page table setup. For 15 * The Linux memory management assumes a three-level page table setup.
16 * s390 31 bit we "fold" the mid level into the top-level page table, so 16 * For s390 64 bit we use up to four of the five levels the hardware
17 * that we physically have the same two-level page table as the s390 mmu 17 * provides (region first tables are not used).
18 * expects in 31 bit mode. For s390 64 bit we use three of the five levels
19 * the hardware provides (region first and region second tables are not
20 * used).
21 * 18 *
22 * The "pgd_xxx()" functions are trivial for a folded two-level 19 * The "pgd_xxx()" functions are trivial for a folded two-level
23 * setup: the pgd is never bad, and a pmd always exists (as it's folded 20 * setup: the pgd is never bad, and a pmd always exists (as it's folded
@@ -101,8 +98,8 @@ extern unsigned long zero_page_mask;
101 98
102#ifndef __ASSEMBLY__ 99#ifndef __ASSEMBLY__
103/* 100/*
104 * The vmalloc and module area will always be on the topmost area of the kernel 101 * The vmalloc and module area will always be on the topmost area of the
105 * mapping. We reserve 96MB (31bit) / 128GB (64bit) for vmalloc and modules. 102 * kernel mapping. We reserve 128GB (64bit) for vmalloc and modules.
106 * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where 103 * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where
107 * modules will reside. That makes sure that inter module branches always 104 * modules will reside. That makes sure that inter module branches always
108 * happen without trampolines and in addition the placement within a 2GB frame 105 * happen without trampolines and in addition the placement within a 2GB frame
@@ -131,38 +128,6 @@ static inline int is_module_addr(void *addr)
131} 128}
132 129
133/* 130/*
134 * A 31 bit pagetable entry of S390 has following format:
135 * | PFRA | | OS |
136 * 0 0IP0
137 * 00000000001111111111222222222233
138 * 01234567890123456789012345678901
139 *
140 * I Page-Invalid Bit: Page is not available for address-translation
141 * P Page-Protection Bit: Store access not possible for page
142 *
143 * A 31 bit segmenttable entry of S390 has following format:
144 * | P-table origin | |PTL
145 * 0 IC
146 * 00000000001111111111222222222233
147 * 01234567890123456789012345678901
148 *
149 * I Segment-Invalid Bit: Segment is not available for address-translation
150 * C Common-Segment Bit: Segment is not private (PoP 3-30)
151 * PTL Page-Table-Length: Page-table length (PTL+1*16 entries -> up to 256)
152 *
153 * The 31 bit segmenttable origin of S390 has following format:
154 *
155 * |S-table origin | | STL |
156 * X **GPS
157 * 00000000001111111111222222222233
158 * 01234567890123456789012345678901
159 *
160 * X Space-Switch event:
161 * G Segment-Invalid Bit: *
162 * P Private-Space Bit: Segment is not private (PoP 3-30)
163 * S Storage-Alteration:
164 * STL Segment-Table-Length: Segment-table length (STL+1*16 entries -> up to 2048)
165 *
166 * A 64 bit pagetable entry of S390 has following format: 131 * A 64 bit pagetable entry of S390 has following format:
167 * | PFRA |0IPC| OS | 132 * | PFRA |0IPC| OS |
168 * 0000000000111111111122222222223333333333444444444455555555556666 133 * 0000000000111111111122222222223333333333444444444455555555556666
@@ -220,7 +185,6 @@ static inline int is_module_addr(void *addr)
220 185
221/* Software bits in the page table entry */ 186/* Software bits in the page table entry */
222#define _PAGE_PRESENT 0x001 /* SW pte present bit */ 187#define _PAGE_PRESENT 0x001 /* SW pte present bit */
223#define _PAGE_TYPE 0x002 /* SW pte type bit */
224#define _PAGE_YOUNG 0x004 /* SW pte young bit */ 188#define _PAGE_YOUNG 0x004 /* SW pte young bit */
225#define _PAGE_DIRTY 0x008 /* SW pte dirty bit */ 189#define _PAGE_DIRTY 0x008 /* SW pte dirty bit */
226#define _PAGE_READ 0x010 /* SW pte read bit */ 190#define _PAGE_READ 0x010 /* SW pte read bit */
@@ -240,31 +204,34 @@ static inline int is_module_addr(void *addr)
240 * table lock held. 204 * table lock held.
241 * 205 *
242 * The following table gives the different possible bit combinations for 206 * The following table gives the different possible bit combinations for
243 * the pte hardware and software bits in the last 12 bits of a pte: 207 * the pte hardware and software bits in the last 12 bits of a pte
208 * (. unassigned bit, x don't care, t swap type):
244 * 209 *
245 * 842100000000 210 * 842100000000
246 * 000084210000 211 * 000084210000
247 * 000000008421 212 * 000000008421
248 * .IR...wrdytp 213 * .IR.uswrdy.p
249 * empty .10...000000 214 * empty .10.00000000
250 * swap .10...xxxx10 215 * swap .11..ttttt.0
251 * file .11...xxxxx0 216 * prot-none, clean, old .11.xx0000.1
252 * prot-none, clean, old .11...000001 217 * prot-none, clean, young .11.xx0001.1
253 * prot-none, clean, young .11...000101 218 * prot-none, dirty, old .10.xx0010.1
254 * prot-none, dirty, old .10...001001 219 * prot-none, dirty, young .10.xx0011.1
255 * prot-none, dirty, young .10...001101 220 * read-only, clean, old .11.xx0100.1
256 * read-only, clean, old .11...010001 221 * read-only, clean, young .01.xx0101.1
257 * read-only, clean, young .01...010101 222 * read-only, dirty, old .11.xx0110.1
258 * read-only, dirty, old .11...011001 223 * read-only, dirty, young .01.xx0111.1
259 * read-only, dirty, young .01...011101 224 * read-write, clean, old .11.xx1100.1
260 * read-write, clean, old .11...110001 225 * read-write, clean, young .01.xx1101.1
261 * read-write, clean, young .01...110101 226 * read-write, dirty, old .10.xx1110.1
262 * read-write, dirty, old .10...111001 227 * read-write, dirty, young .00.xx1111.1
263 * read-write, dirty, young .00...111101 228 * HW-bits: R read-only, I invalid
229 * SW-bits: p present, y young, d dirty, r read, w write, s special,
230 * u unused, l large
264 * 231 *
265 * pte_present is true for the bit pattern .xx...xxxxx1, (pte & 0x001) == 0x001 232 * pte_none is true for the bit pattern .10.00000000, pte == 0x400
266 * pte_none is true for the bit pattern .10...xxxx00, (pte & 0x603) == 0x400 233 * pte_swap is true for the bit pattern .11..ooooo.0, (pte & 0x201) == 0x200
267 * pte_swap is true for the bit pattern .10...xxxx10, (pte & 0x603) == 0x402 234 * pte_present is true for the bit pattern .xx.xxxxxx.1, (pte & 0x001) == 0x001
268 */ 235 */
269 236
270/* Bits in the segment/region table address-space-control-element */ 237/* Bits in the segment/region table address-space-control-element */
@@ -335,6 +302,8 @@ static inline int is_module_addr(void *addr)
335 * read-write, dirty, young 11..0...0...11 302 * read-write, dirty, young 11..0...0...11
336 * The segment table origin is used to distinguish empty (origin==0) from 303 * The segment table origin is used to distinguish empty (origin==0) from
337 * read-write, old segment table entries (origin!=0) 304 * read-write, old segment table entries (origin!=0)
305 * HW-bits: R read-only, I invalid
306 * SW-bits: y young, d dirty, r read, w write
338 */ 307 */
339 308
340#define _SEGMENT_ENTRY_SPLIT_BIT 11 /* THP splitting bit number */ 309#define _SEGMENT_ENTRY_SPLIT_BIT 11 /* THP splitting bit number */
@@ -423,6 +392,15 @@ static inline int mm_has_pgste(struct mm_struct *mm)
423 return 0; 392 return 0;
424} 393}
425 394
395static inline int mm_alloc_pgste(struct mm_struct *mm)
396{
397#ifdef CONFIG_PGSTE
398 if (unlikely(mm->context.alloc_pgste))
399 return 1;
400#endif
401 return 0;
402}
403
426/* 404/*
427 * In the case that a guest uses storage keys 405 * In the case that a guest uses storage keys
428 * faults should no longer be backed by zero pages 406 * faults should no longer be backed by zero pages
@@ -582,10 +560,9 @@ static inline int pte_none(pte_t pte)
582 560
583static inline int pte_swap(pte_t pte) 561static inline int pte_swap(pte_t pte)
584{ 562{
585 /* Bit pattern: (pte & 0x603) == 0x402 */ 563 /* Bit pattern: (pte & 0x201) == 0x200 */
586 return (pte_val(pte) & (_PAGE_INVALID | _PAGE_PROTECT | 564 return (pte_val(pte) & (_PAGE_PROTECT | _PAGE_PRESENT))
587 _PAGE_TYPE | _PAGE_PRESENT)) 565 == _PAGE_PROTECT;
588 == (_PAGE_INVALID | _PAGE_TYPE);
589} 566}
590 567
591static inline int pte_special(pte_t pte) 568static inline int pte_special(pte_t pte)
@@ -1586,51 +1563,51 @@ static inline int has_transparent_hugepage(void)
1586#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1563#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1587 1564
1588/* 1565/*
1589 * 31 bit swap entry format:
1590 * A page-table entry has some bits we have to treat in a special way.
1591 * Bits 0, 20 and bit 23 have to be zero, otherwise an specification
1592 * exception will occur instead of a page translation exception. The
1593 * specifiation exception has the bad habit not to store necessary
1594 * information in the lowcore.
1595 * Bits 21, 22, 30 and 31 are used to indicate the page type.
1596 * A swap pte is indicated by bit pattern (pte & 0x603) == 0x402
1597 * This leaves the bits 1-19 and bits 24-29 to store type and offset.
1598 * We use the 5 bits from 25-29 for the type and the 20 bits from 1-19
1599 * plus 24 for the offset.
1600 * 0| offset |0110|o|type |00|
1601 * 0 0000000001111111111 2222 2 22222 33
1602 * 0 1234567890123456789 0123 4 56789 01
1603 *
1604 * 64 bit swap entry format: 1566 * 64 bit swap entry format:
1605 * A page-table entry has some bits we have to treat in a special way. 1567 * A page-table entry has some bits we have to treat in a special way.
1606 * Bits 52 and bit 55 have to be zero, otherwise an specification 1568 * Bits 52 and bit 55 have to be zero, otherwise an specification
1607 * exception will occur instead of a page translation exception. The 1569 * exception will occur instead of a page translation exception. The
1608 * specifiation exception has the bad habit not to store necessary 1570 * specifiation exception has the bad habit not to store necessary
1609 * information in the lowcore. 1571 * information in the lowcore.
1610 * Bits 53, 54, 62 and 63 are used to indicate the page type. 1572 * Bits 54 and 63 are used to indicate the page type.
1611 * A swap pte is indicated by bit pattern (pte & 0x603) == 0x402 1573 * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
1612 * This leaves the bits 0-51 and bits 56-61 to store type and offset. 1574 * This leaves the bits 0-51 and bits 56-62 to store type and offset.
1613 * We use the 5 bits from 57-61 for the type and the 53 bits from 0-51 1575 * We use the 5 bits from 57-61 for the type and the 52 bits from 0-51
1614 * plus 56 for the offset. 1576 * for the offset.
1615 * | offset |0110|o|type |00| 1577 * | offset |01100|type |00|
1616 * 0000000000111111111122222222223333333333444444444455 5555 5 55566 66 1578 * |0000000000111111111122222222223333333333444444444455|55555|55566|66|
1617 * 0123456789012345678901234567890123456789012345678901 2345 6 78901 23 1579 * |0123456789012345678901234567890123456789012345678901|23456|78901|23|
1618 */ 1580 */
1619 1581
1620#define __SWP_OFFSET_MASK (~0UL >> 11) 1582#define __SWP_OFFSET_MASK ((1UL << 52) - 1)
1583#define __SWP_OFFSET_SHIFT 12
1584#define __SWP_TYPE_MASK ((1UL << 5) - 1)
1585#define __SWP_TYPE_SHIFT 2
1621 1586
1622static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) 1587static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1623{ 1588{
1624 pte_t pte; 1589 pte_t pte;
1625 offset &= __SWP_OFFSET_MASK; 1590
1626 pte_val(pte) = _PAGE_INVALID | _PAGE_TYPE | ((type & 0x1f) << 2) | 1591 pte_val(pte) = _PAGE_INVALID | _PAGE_PROTECT;
1627 ((offset & 1UL) << 7) | ((offset & ~1UL) << 11); 1592 pte_val(pte) |= (offset & __SWP_OFFSET_MASK) << __SWP_OFFSET_SHIFT;
1593 pte_val(pte) |= (type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT;
1628 return pte; 1594 return pte;
1629} 1595}
1630 1596
1631#define __swp_type(entry) (((entry).val >> 2) & 0x1f) 1597static inline unsigned long __swp_type(swp_entry_t entry)
1632#define __swp_offset(entry) (((entry).val >> 11) | (((entry).val >> 7) & 1)) 1598{
1633#define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) }) 1599 return (entry.val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK;
1600}
1601
1602static inline unsigned long __swp_offset(swp_entry_t entry)
1603{
1604 return (entry.val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK;
1605}
1606
1607static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset)
1608{
1609 return (swp_entry_t) { pte_val(mk_swap_pte(type, offset)) };
1610}
1634 1611
1635#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) 1612#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1636#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) 1613#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index 210ffede0153..e617e74b7be2 100644
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -14,20 +14,23 @@ static inline pmd_t __pte_to_pmd(pte_t pte)
14 14
15 /* 15 /*
16 * Convert encoding pte bits pmd bits 16 * Convert encoding pte bits pmd bits
17 * .IR...wrdytp dy..R...I...wr 17 * lIR.uswrdy.p dy..R...I...wr
18 * empty .10...000000 -> 00..0...1...00 18 * empty 010.000000.0 -> 00..0...1...00
19 * prot-none, clean, old .11...000001 -> 00..1...1...00 19 * prot-none, clean, old 111.000000.1 -> 00..1...1...00
20 * prot-none, clean, young .11...000101 -> 01..1...1...00 20 * prot-none, clean, young 111.000001.1 -> 01..1...1...00
21 * prot-none, dirty, old .10...001001 -> 10..1...1...00 21 * prot-none, dirty, old 111.000010.1 -> 10..1...1...00
22 * prot-none, dirty, young .10...001101 -> 11..1...1...00 22 * prot-none, dirty, young 111.000011.1 -> 11..1...1...00
23 * read-only, clean, old .11...010001 -> 00..1...1...01 23 * read-only, clean, old 111.000100.1 -> 00..1...1...01
24 * read-only, clean, young .01...010101 -> 01..1...0...01 24 * read-only, clean, young 101.000101.1 -> 01..1...0...01
25 * read-only, dirty, old .11...011001 -> 10..1...1...01 25 * read-only, dirty, old 111.000110.1 -> 10..1...1...01
26 * read-only, dirty, young .01...011101 -> 11..1...0...01 26 * read-only, dirty, young 101.000111.1 -> 11..1...0...01
27 * read-write, clean, old .11...110001 -> 00..0...1...11 27 * read-write, clean, old 111.001100.1 -> 00..1...1...11
28 * read-write, clean, young .01...110101 -> 01..0...0...11 28 * read-write, clean, young 101.001101.1 -> 01..1...0...11
29 * read-write, dirty, old .10...111001 -> 10..0...1...11 29 * read-write, dirty, old 110.001110.1 -> 10..0...1...11
30 * read-write, dirty, young .00...111101 -> 11..0...0...11 30 * read-write, dirty, young 100.001111.1 -> 11..0...0...11
31 * HW-bits: R read-only, I invalid
32 * SW-bits: p present, y young, d dirty, r read, w write, s special,
33 * u unused, l large
31 */ 34 */
32 if (pte_present(pte)) { 35 if (pte_present(pte)) {
33 pmd_val(pmd) = pte_val(pte) & PAGE_MASK; 36 pmd_val(pmd) = pte_val(pte) & PAGE_MASK;
@@ -48,20 +51,23 @@ static inline pte_t __pmd_to_pte(pmd_t pmd)
48 51
49 /* 52 /*
50 * Convert encoding pmd bits pte bits 53 * Convert encoding pmd bits pte bits
51 * dy..R...I...wr .IR...wrdytp 54 * dy..R...I...wr lIR.uswrdy.p
52 * empty 00..0...1...00 -> .10...001100 55 * empty 00..0...1...00 -> 010.000000.0
53 * prot-none, clean, old 00..0...1...00 -> .10...000001 56 * prot-none, clean, old 00..1...1...00 -> 111.000000.1
54 * prot-none, clean, young 01..0...1...00 -> .10...000101 57 * prot-none, clean, young 01..1...1...00 -> 111.000001.1
55 * prot-none, dirty, old 10..0...1...00 -> .10...001001 58 * prot-none, dirty, old 10..1...1...00 -> 111.000010.1
56 * prot-none, dirty, young 11..0...1...00 -> .10...001101 59 * prot-none, dirty, young 11..1...1...00 -> 111.000011.1
57 * read-only, clean, old 00..1...1...01 -> .11...010001 60 * read-only, clean, old 00..1...1...01 -> 111.000100.1
58 * read-only, clean, young 01..1...1...01 -> .11...010101 61 * read-only, clean, young 01..1...0...01 -> 101.000101.1
59 * read-only, dirty, old 10..1...1...01 -> .11...011001 62 * read-only, dirty, old 10..1...1...01 -> 111.000110.1
60 * read-only, dirty, young 11..1...1...01 -> .11...011101 63 * read-only, dirty, young 11..1...0...01 -> 101.000111.1
61 * read-write, clean, old 00..0...1...11 -> .10...110001 64 * read-write, clean, old 00..1...1...11 -> 111.001100.1
62 * read-write, clean, young 01..0...1...11 -> .10...110101 65 * read-write, clean, young 01..1...0...11 -> 101.001101.1
63 * read-write, dirty, old 10..0...1...11 -> .10...111001 66 * read-write, dirty, old 10..0...1...11 -> 110.001110.1
64 * read-write, dirty, young 11..0...1...11 -> .10...111101 67 * read-write, dirty, young 11..0...0...11 -> 100.001111.1
68 * HW-bits: R read-only, I invalid
69 * SW-bits: p present, y young, d dirty, r read, w write, s special,
70 * u unused, l large
65 */ 71 */
66 if (pmd_present(pmd)) { 72 if (pmd_present(pmd)) {
67 pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE; 73 pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE;
@@ -70,8 +76,8 @@ static inline pte_t __pmd_to_pte(pmd_t pmd)
70 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4; 76 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4;
71 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5; 77 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5;
72 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT); 78 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT);
73 pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10; 79 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) >> 10;
74 pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10; 80 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) >> 10;
75 } else 81 } else
76 pte_val(pte) = _PAGE_INVALID; 82 pte_val(pte) = _PAGE_INVALID;
77 return pte; 83 return pte;
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 33f589459113..b33f66110ca9 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -18,6 +18,7 @@
18#include <linux/rcupdate.h> 18#include <linux/rcupdate.h>
19#include <linux/slab.h> 19#include <linux/slab.h>
20#include <linux/swapops.h> 20#include <linux/swapops.h>
21#include <linux/sysctl.h>
21#include <linux/ksm.h> 22#include <linux/ksm.h>
22#include <linux/mman.h> 23#include <linux/mman.h>
23 24
@@ -920,6 +921,40 @@ unsigned long get_guest_storage_key(struct mm_struct *mm, unsigned long addr)
920} 921}
921EXPORT_SYMBOL(get_guest_storage_key); 922EXPORT_SYMBOL(get_guest_storage_key);
922 923
924static int page_table_allocate_pgste_min = 0;
925static int page_table_allocate_pgste_max = 1;
926int page_table_allocate_pgste = 0;
927EXPORT_SYMBOL(page_table_allocate_pgste);
928
929static struct ctl_table page_table_sysctl[] = {
930 {
931 .procname = "allocate_pgste",
932 .data = &page_table_allocate_pgste,
933 .maxlen = sizeof(int),
934 .mode = S_IRUGO | S_IWUSR,
935 .proc_handler = proc_dointvec,
936 .extra1 = &page_table_allocate_pgste_min,
937 .extra2 = &page_table_allocate_pgste_max,
938 },
939 { }
940};
941
942static struct ctl_table page_table_sysctl_dir[] = {
943 {
944 .procname = "vm",
945 .maxlen = 0,
946 .mode = 0555,
947 .child = page_table_sysctl,
948 },
949 { }
950};
951
952static int __init page_table_register_sysctl(void)
953{
954 return register_sysctl_table(page_table_sysctl_dir) ? 0 : -ENOMEM;
955}
956__initcall(page_table_register_sysctl);
957
923#else /* CONFIG_PGSTE */ 958#else /* CONFIG_PGSTE */
924 959
925static inline int page_table_with_pgste(struct page *page) 960static inline int page_table_with_pgste(struct page *page)
@@ -963,7 +998,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
963 struct page *uninitialized_var(page); 998 struct page *uninitialized_var(page);
964 unsigned int mask, bit; 999 unsigned int mask, bit;
965 1000
966 if (mm_has_pgste(mm)) 1001 if (mm_alloc_pgste(mm))
967 return page_table_alloc_pgste(mm); 1002 return page_table_alloc_pgste(mm);
968 /* Allocate fragments of a 4K page as 1K/2K page table */ 1003 /* Allocate fragments of a 4K page as 1K/2K page table */
969 spin_lock_bh(&mm->context.list_lock); 1004 spin_lock_bh(&mm->context.list_lock);
@@ -1165,116 +1200,25 @@ static inline void thp_split_mm(struct mm_struct *mm)
1165} 1200}
1166#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1201#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1167 1202
1168static unsigned long page_table_realloc_pmd(struct mmu_gather *tlb,
1169 struct mm_struct *mm, pud_t *pud,
1170 unsigned long addr, unsigned long end)
1171{
1172 unsigned long next, *table, *new;
1173 struct page *page;
1174 spinlock_t *ptl;
1175 pmd_t *pmd;
1176
1177 pmd = pmd_offset(pud, addr);
1178 do {
1179 next = pmd_addr_end(addr, end);
1180again:
1181 if (pmd_none_or_clear_bad(pmd))
1182 continue;
1183 table = (unsigned long *) pmd_deref(*pmd);
1184 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
1185 if (page_table_with_pgste(page))
1186 continue;
1187 /* Allocate new page table with pgstes */
1188 new = page_table_alloc_pgste(mm);
1189 if (!new)
1190 return -ENOMEM;
1191
1192 ptl = pmd_lock(mm, pmd);
1193 if (likely((unsigned long *) pmd_deref(*pmd) == table)) {
1194 /* Nuke pmd entry pointing to the "short" page table */
1195 pmdp_flush_lazy(mm, addr, pmd);
1196 pmd_clear(pmd);
1197 /* Copy ptes from old table to new table */
1198 memcpy(new, table, PAGE_SIZE/2);
1199 clear_table(table, _PAGE_INVALID, PAGE_SIZE/2);
1200 /* Establish new table */
1201 pmd_populate(mm, pmd, (pte_t *) new);
1202 /* Free old table with rcu, there might be a walker! */
1203 page_table_free_rcu(tlb, table, addr);
1204 new = NULL;
1205 }
1206 spin_unlock(ptl);
1207 if (new) {
1208 page_table_free_pgste(new);
1209 goto again;
1210 }
1211 } while (pmd++, addr = next, addr != end);
1212
1213 return addr;
1214}
1215
1216static unsigned long page_table_realloc_pud(struct mmu_gather *tlb,
1217 struct mm_struct *mm, pgd_t *pgd,
1218 unsigned long addr, unsigned long end)
1219{
1220 unsigned long next;
1221 pud_t *pud;
1222
1223 pud = pud_offset(pgd, addr);
1224 do {
1225 next = pud_addr_end(addr, end);
1226 if (pud_none_or_clear_bad(pud))
1227 continue;
1228 next = page_table_realloc_pmd(tlb, mm, pud, addr, next);
1229 if (unlikely(IS_ERR_VALUE(next)))
1230 return next;
1231 } while (pud++, addr = next, addr != end);
1232
1233 return addr;
1234}
1235
1236static unsigned long page_table_realloc(struct mmu_gather *tlb, struct mm_struct *mm,
1237 unsigned long addr, unsigned long end)
1238{
1239 unsigned long next;
1240 pgd_t *pgd;
1241
1242 pgd = pgd_offset(mm, addr);
1243 do {
1244 next = pgd_addr_end(addr, end);
1245 if (pgd_none_or_clear_bad(pgd))
1246 continue;
1247 next = page_table_realloc_pud(tlb, mm, pgd, addr, next);
1248 if (unlikely(IS_ERR_VALUE(next)))
1249 return next;
1250 } while (pgd++, addr = next, addr != end);
1251
1252 return 0;
1253}
1254
1255/* 1203/*
1256 * switch on pgstes for its userspace process (for kvm) 1204 * switch on pgstes for its userspace process (for kvm)
1257 */ 1205 */
1258int s390_enable_sie(void) 1206int s390_enable_sie(void)
1259{ 1207{
1260 struct task_struct *tsk = current; 1208 struct mm_struct *mm = current->mm;
1261 struct mm_struct *mm = tsk->mm;
1262 struct mmu_gather tlb;
1263 1209
1264 /* Do we have pgstes? if yes, we are done */ 1210 /* Do we have pgstes? if yes, we are done */
1265 if (mm_has_pgste(tsk->mm)) 1211 if (mm_has_pgste(mm))
1266 return 0; 1212 return 0;
1267 1213 /* Fail if the page tables are 2K */
1214 if (!mm_alloc_pgste(mm))
1215 return -EINVAL;
1268 down_write(&mm->mmap_sem); 1216 down_write(&mm->mmap_sem);
1217 mm->context.has_pgste = 1;
1269 /* split thp mappings and disable thp for future mappings */ 1218 /* split thp mappings and disable thp for future mappings */
1270 thp_split_mm(mm); 1219 thp_split_mm(mm);
1271 /* Reallocate the page tables with pgstes */
1272 tlb_gather_mmu(&tlb, mm, 0, TASK_SIZE);
1273 if (!page_table_realloc(&tlb, mm, 0, TASK_SIZE))
1274 mm->context.has_pgste = 1;
1275 tlb_finish_mmu(&tlb, 0, TASK_SIZE);
1276 up_write(&mm->mmap_sem); 1220 up_write(&mm->mmap_sem);
1277 return mm->context.has_pgste ? 0 : -ENOMEM; 1221 return 0;
1278} 1222}
1279EXPORT_SYMBOL_GPL(s390_enable_sie); 1223EXPORT_SYMBOL_GPL(s390_enable_sie);
1280 1224
diff --git a/drivers/s390/char/con3215.c b/drivers/s390/char/con3215.c
index c43aca69fb30..0fc3fe5fd5b8 100644
--- a/drivers/s390/char/con3215.c
+++ b/drivers/s390/char/con3215.c
@@ -667,6 +667,8 @@ static struct raw3215_info *raw3215_alloc_info(void)
667 info->buffer = kzalloc(RAW3215_BUFFER_SIZE, GFP_KERNEL | GFP_DMA); 667 info->buffer = kzalloc(RAW3215_BUFFER_SIZE, GFP_KERNEL | GFP_DMA);
668 info->inbuf = kzalloc(RAW3215_INBUF_SIZE, GFP_KERNEL | GFP_DMA); 668 info->inbuf = kzalloc(RAW3215_INBUF_SIZE, GFP_KERNEL | GFP_DMA);
669 if (!info->buffer || !info->inbuf) { 669 if (!info->buffer || !info->inbuf) {
670 kfree(info->inbuf);
671 kfree(info->buffer);
670 kfree(info); 672 kfree(info);
671 return NULL; 673 return NULL;
672 } 674 }
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index e60a745ac198..e804306ef5e8 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -40,6 +40,10 @@
40#error KEXEC_CONTROL_MEMORY_LIMIT not defined 40#error KEXEC_CONTROL_MEMORY_LIMIT not defined
41#endif 41#endif
42 42
43#ifndef KEXEC_CONTROL_MEMORY_GFP
44#define KEXEC_CONTROL_MEMORY_GFP GFP_KERNEL
45#endif
46
43#ifndef KEXEC_CONTROL_PAGE_SIZE 47#ifndef KEXEC_CONTROL_PAGE_SIZE
44#error KEXEC_CONTROL_PAGE_SIZE not defined 48#error KEXEC_CONTROL_PAGE_SIZE not defined
45#endif 49#endif
diff --git a/kernel/kexec.c b/kernel/kexec.c
index 38c25b1f2fd5..7a36fdcca5bf 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -707,7 +707,7 @@ static struct page *kimage_alloc_normal_control_pages(struct kimage *image,
707 do { 707 do {
708 unsigned long pfn, epfn, addr, eaddr; 708 unsigned long pfn, epfn, addr, eaddr;
709 709
710 pages = kimage_alloc_pages(GFP_KERNEL, order); 710 pages = kimage_alloc_pages(KEXEC_CONTROL_MEMORY_GFP, order);
711 if (!pages) 711 if (!pages)
712 break; 712 break;
713 pfn = page_to_pfn(pages); 713 pfn = page_to_pfn(pages);
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-31 14:34:25 -0400