1 files changed, 154 insertions, 35 deletions
diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
index d32cfb27a479..4de8f5b3d476 100644
--- a/arch/x86/kernel/relocate_kernel_64.S
+++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -19,29 +19,77 @@
 #define PTR(x) (x << 3)
 #define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)
+/*
+ * control_page + KEXEC_CONTROL_CODE_MAX_SIZE
+ * ~ control_page + PAGE_SIZE are used as data storage and stack for
+ * jumping back
+ */
+#define DATA(offset)            (KEXEC_CONTROL_CODE_MAX_SIZE+(offset))
+/* Minimal CPU state */
+#define RSP                     DATA(0x0)
+#define CR0                     DATA(0x8)
+#define CR3                     DATA(0x10)
+#define CR4                     DATA(0x18)
+/* other data */
+#define CP_PA_TABLE_PAGE        DATA(0x20)
+#define CP_PA_SWAP_PAGE         DATA(0x28)
+#define CP_PA_BACKUP_PAGES_MAP  DATA(0x30)
        .text
        .align PAGE_SIZE
        .code64
        .globl relocate_kernel
 relocate_kernel:
-        /* %rdi indirection_page
+        /*
+         * %rdi indirection_page
         * %rsi page_list
         * %rdx start address
+         * %rcx preserve_context
         */
+        /* Save the CPU context, used for jumping back */
+        pushq %rbx
+        pushq %rbp
+        pushq %r12
+        pushq %r13
+        pushq %r14
+        pushq %r15
+        pushf
+        movq    PTR(VA_CONTROL_PAGE)(%rsi), %r11
+        movq    %rsp, RSP(%r11)
+        movq    %cr0, %rax
+        movq    %rax, CR0(%r11)
+        movq    %cr3, %rax
+        movq    %rax, CR3(%r11)
+        movq    %cr4, %rax
+        movq    %rax, CR4(%r11)
        /* zero out flags, and disable interrupts */
        pushq $0
        popfq
-        /* get physical address of control page now */
+        /*
-        /* this is impossible after page table switch */
+         * get physical address of control page now
+         * this is impossible after page table switch
+         */
        movq    PTR(PA_CONTROL_PAGE)(%rsi), %r8
        /* get physical address of page table now too */
-        movq    PTR(PA_TABLE_PAGE)(%rsi), %rcx
+        movq    PTR(PA_TABLE_PAGE)(%rsi), %r9
+        /* get physical address of swap page now */
+        movq    PTR(PA_SWAP_PAGE)(%rsi), %r10
+        /* save some information for jumping back */
+        movq    %r9, CP_PA_TABLE_PAGE(%r11)
+        movq    %r10, CP_PA_SWAP_PAGE(%r11)
+        movq    %rdi, CP_PA_BACKUP_PAGES_MAP(%r11)
        /* Switch to the identity mapped page tables */
-        movq    %rcx, %cr3
+        movq    %r9, %cr3
        /* setup a new stack at the end of the physical control page */
        lea     PAGE_SIZE(%r8), %rsp
@@ -55,7 +103,8 @@ identity_mapped:
        /* store the start address on the stack */
        pushq   %rdx
-        /* Set cr0 to a known state:
+        /*
+         * Set cr0 to a known state:
         *  - Paging enabled
         *  - Alignment check disabled
         *  - Write protect disabled
@@ -68,7 +117,8 @@ identity_mapped:
        orl     $(X86_CR0_PG | X86_CR0_PE), %eax
        movq    %rax, %cr0
-        /* Set cr4 to a known state:
+        /*
+         * Set cr4 to a known state:
         *  - physical address extension enabled
         */
        movq    $X86_CR4_PAE, %rax
@@ -78,9 +128,87 @@ identity_mapped:
 1:
        /* Flush the TLB (needed?) */
-        movq    %rcx, %cr3
+        movq    %r9, %cr3
+        movq    %rcx, %r11
+        call    swap_pages
+        /*
+         * To be certain of avoiding problems with self-modifying code
+         * I need to execute a serializing instruction here.
+         * So I flush the TLB by reloading %cr3 here, it's handy,
+         * and not processor dependent.
+         */
+        movq    %cr3, %rax
+        movq    %rax, %cr3
+        /*
+         * set all of the registers to known values
+         * leave %rsp alone
+         */
+        testq   %r11, %r11
+        jnz 1f
+        xorq    %rax, %rax
+        xorq    %rbx, %rbx
+        xorq    %rcx, %rcx
+        xorq    %rdx, %rdx
+        xorq    %rsi, %rsi
+        xorq    %rdi, %rdi
+        xorq    %rbp, %rbp
+        xorq    %r8,  %r8
+        xorq    %r9,  %r9
+        xorq    %r10, %r9
+        xorq    %r11, %r11
+        xorq    %r12, %r12
+        xorq    %r13, %r13
+        xorq    %r14, %r14
+        xorq    %r15, %r15
+        ret
+1:
+        popq    %rdx
+        leaq    PAGE_SIZE(%r10), %rsp
+        call    *%rdx
+        /* get the re-entry point of the peer system */
+        movq    0(%rsp), %rbp
+        call    1f
+1:
+        popq    %r8
+        subq    $(1b - relocate_kernel), %r8
+        movq    CP_PA_SWAP_PAGE(%r8), %r10
+        movq    CP_PA_BACKUP_PAGES_MAP(%r8), %rdi
+        movq    CP_PA_TABLE_PAGE(%r8), %rax
+        movq    %rax, %cr3
+        lea     PAGE_SIZE(%r8), %rsp
+        call    swap_pages
+        movq    $virtual_mapped, %rax
+        pushq   %rax
+        ret
+virtual_mapped:
+        movq    RSP(%r8), %rsp
+        movq    CR4(%r8), %rax
+        movq    %rax, %cr4
+        movq    CR3(%r8), %rax
+        movq    CR0(%r8), %r8
+        movq    %rax, %cr3
+        movq    %r8, %cr0
+        movq    %rbp, %rax
+        popf
+        popq    %r15
+        popq    %r14
+        popq    %r13
+        popq    %r12
+        popq    %rbp
+        popq    %rbx
+        ret
        /* Do the copies */
+swap_pages:
        movq    %rdi, %rcx      /* Put the page_list in %rcx */
        xorq    %rdi, %rdi
        xorq    %rsi, %rsi
@@ -112,36 +240,27 @@ identity_mapped:
        movq    %rcx,   %rsi  /* For ever source page do a copy */
        andq    $0xfffffffffffff000, %rsi
+        movq    %rdi, %rdx
+        movq    %rsi, %rax
+        movq    %r10, %rdi
        movq    $512,   %rcx
        rep ; movsq
-        jmp     0b
-3:
-        /* To be certain of avoiding problems with self-modifying code
-         * I need to execute a serializing instruction here.
-         * So I flush the TLB by reloading %cr3 here, it's handy,
-         * and not processor dependent.
-         */
-        movq    %cr3, %rax
-        movq    %rax, %cr3
-        /* set all of the registers to known values */
+        movq    %rax, %rdi
-        /* leave %rsp alone */
+        movq    %rdx, %rsi
+        movq    $512,   %rcx
+        rep ; movsq
-        xorq    %rax, %rax
+        movq    %rdx, %rdi
-        xorq    %rbx, %rbx
+        movq    %r10, %rsi
-        xorq    %rcx, %rcx
+        movq    $512,   %rcx
-        xorq    %rdx, %rdx
+        rep ; movsq
-        xorq    %rsi, %rsi
-        xorq    %rdi, %rdi
-        xorq    %rbp, %rbp
-        xorq    %r8,  %r8
-        xorq    %r9,  %r9
-        xorq    %r10, %r9
-        xorq    %r11, %r11
-        xorq    %r12, %r12
-        xorq    %r13, %r13
-        xorq    %r14, %r14
-        xorq    %r15, %r15
+        lea     PAGE_SIZE(%rax), %rsi
+        jmp     0b
+3:
        ret
+        .globl kexec_control_code_size
+.set kexec_control_code_size, . - relocate_kernel

diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index d32cfb27a479..4de8f5b3d476 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -19,29 +19,77 @@
19	#define PTR(x) (x << 3)	19	#define PTR(x) (x << 3)
20	#define PAGE_ATTR (_PAGE_PRESENT \| _PAGE_RW \| _PAGE_ACCESSED \| _PAGE_DIRTY)	20	#define PAGE_ATTR (_PAGE_PRESENT \| _PAGE_RW \| _PAGE_ACCESSED \| _PAGE_DIRTY)
21		21
		22	/*
		23	* control_page + KEXEC_CONTROL_CODE_MAX_SIZE
		24	* ~ control_page + PAGE_SIZE are used as data storage and stack for
		25	* jumping back
		26	*/
		27	#define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset))
		28
		29	/* Minimal CPU state */
		30	#define RSP DATA(0x0)
		31	#define CR0 DATA(0x8)
		32	#define CR3 DATA(0x10)
		33	#define CR4 DATA(0x18)
		34
		35	/* other data */
		36	#define CP_PA_TABLE_PAGE DATA(0x20)
		37	#define CP_PA_SWAP_PAGE DATA(0x28)
		38	#define CP_PA_BACKUP_PAGES_MAP DATA(0x30)
		39
22	.text	40	.text
23	.align PAGE_SIZE	41	.align PAGE_SIZE
24	.code64	42	.code64
25	.globl relocate_kernel	43	.globl relocate_kernel
26	relocate_kernel:	44	relocate_kernel:
27	/* %rdi indirection_page	45	/*
		46	* %rdi indirection_page
28	* %rsi page_list	47	* %rsi page_list
29	* %rdx start address	48	* %rdx start address
		49	* %rcx preserve_context
30	*/	50	*/
31		51
		52	/* Save the CPU context, used for jumping back */
		53	pushq %rbx
		54	pushq %rbp
		55	pushq %r12
		56	pushq %r13
		57	pushq %r14
		58	pushq %r15
		59	pushf
		60
		61	movq PTR(VA_CONTROL_PAGE)(%rsi), %r11
		62	movq %rsp, RSP(%r11)
		63	movq %cr0, %rax
		64	movq %rax, CR0(%r11)
		65	movq %cr3, %rax
		66	movq %rax, CR3(%r11)
		67	movq %cr4, %rax
		68	movq %rax, CR4(%r11)
		69
32	/* zero out flags, and disable interrupts */	70	/* zero out flags, and disable interrupts */
33	pushq $0	71	pushq $0
34	popfq	72	popfq
35		73
36	/* get physical address of control page now */	74	/*
37	/* this is impossible after page table switch */	75	* get physical address of control page now
		76	* this is impossible after page table switch
		77	*/
38	movq PTR(PA_CONTROL_PAGE)(%rsi), %r8	78	movq PTR(PA_CONTROL_PAGE)(%rsi), %r8
39		79
40	/* get physical address of page table now too */	80	/* get physical address of page table now too */
41	movq PTR(PA_TABLE_PAGE)(%rsi), %rcx	81	movq PTR(PA_TABLE_PAGE)(%rsi), %r9
		82
		83	/* get physical address of swap page now */
		84	movq PTR(PA_SWAP_PAGE)(%rsi), %r10
		85
		86	/* save some information for jumping back */
		87	movq %r9, CP_PA_TABLE_PAGE(%r11)
		88	movq %r10, CP_PA_SWAP_PAGE(%r11)
		89	movq %rdi, CP_PA_BACKUP_PAGES_MAP(%r11)
42		90
43	/* Switch to the identity mapped page tables */	91	/* Switch to the identity mapped page tables */
44	movq %rcx, %cr3	92	movq %r9, %cr3
45		93
46	/* setup a new stack at the end of the physical control page */	94	/* setup a new stack at the end of the physical control page */
47	lea PAGE_SIZE(%r8), %rsp	95	lea PAGE_SIZE(%r8), %rsp
@@ -55,7 +103,8 @@ identity_mapped:
55	/* store the start address on the stack */	103	/* store the start address on the stack */
56	pushq %rdx	104	pushq %rdx
57		105
58	/* Set cr0 to a known state:	106	/*
		107	* Set cr0 to a known state:
59	* - Paging enabled	108	* - Paging enabled
60	* - Alignment check disabled	109	* - Alignment check disabled
61	* - Write protect disabled	110	* - Write protect disabled
@@ -68,7 +117,8 @@ identity_mapped:
68	orl $(X86_CR0_PG \| X86_CR0_PE), %eax	117	orl $(X86_CR0_PG \| X86_CR0_PE), %eax
69	movq %rax, %cr0	118	movq %rax, %cr0
70		119
71	/* Set cr4 to a known state:	120	/*
		121	* Set cr4 to a known state:
72	* - physical address extension enabled	122	* - physical address extension enabled
73	*/	123	*/
74	movq $X86_CR4_PAE, %rax	124	movq $X86_CR4_PAE, %rax
@@ -78,9 +128,87 @@ identity_mapped:
78	1:	128	1:
79		129
80	/* Flush the TLB (needed?) */	130	/* Flush the TLB (needed?) */
81	movq %rcx, %cr3	131	movq %r9, %cr3
		132
		133	movq %rcx, %r11
		134	call swap_pages
		135
		136	/*
		137	* To be certain of avoiding problems with self-modifying code
		138	* I need to execute a serializing instruction here.
		139	* So I flush the TLB by reloading %cr3 here, it's handy,
		140	* and not processor dependent.
		141	*/
		142	movq %cr3, %rax
		143	movq %rax, %cr3
		144
		145	/*
		146	* set all of the registers to known values
		147	* leave %rsp alone
		148	*/
		149
		150	testq %r11, %r11
		151	jnz 1f
		152	xorq %rax, %rax
		153	xorq %rbx, %rbx
		154	xorq %rcx, %rcx
		155	xorq %rdx, %rdx
		156	xorq %rsi, %rsi
		157	xorq %rdi, %rdi
		158	xorq %rbp, %rbp
		159	xorq %r8, %r8
		160	xorq %r9, %r9
		161	xorq %r10, %r9
		162	xorq %r11, %r11
		163	xorq %r12, %r12
		164	xorq %r13, %r13
		165	xorq %r14, %r14
		166	xorq %r15, %r15
		167
		168	ret
		169
		170	1:
		171	popq %rdx
		172	leaq PAGE_SIZE(%r10), %rsp
		173	call *%rdx
		174
		175	/* get the re-entry point of the peer system */
		176	movq 0(%rsp), %rbp
		177	call 1f
		178	1:
		179	popq %r8
		180	subq $(1b - relocate_kernel), %r8
		181	movq CP_PA_SWAP_PAGE(%r8), %r10
		182	movq CP_PA_BACKUP_PAGES_MAP(%r8), %rdi
		183	movq CP_PA_TABLE_PAGE(%r8), %rax
		184	movq %rax, %cr3
		185	lea PAGE_SIZE(%r8), %rsp
		186	call swap_pages
		187	movq $virtual_mapped, %rax
		188	pushq %rax
		189	ret
		190
		191	virtual_mapped:
		192	movq RSP(%r8), %rsp
		193	movq CR4(%r8), %rax
		194	movq %rax, %cr4
		195	movq CR3(%r8), %rax
		196	movq CR0(%r8), %r8
		197	movq %rax, %cr3
		198	movq %r8, %cr0
		199	movq %rbp, %rax
		200
		201	popf
		202	popq %r15
		203	popq %r14
		204	popq %r13
		205	popq %r12
		206	popq %rbp
		207	popq %rbx
		208	ret
82		209
83	/* Do the copies */	210	/* Do the copies */
		211	swap_pages:
84	movq %rdi, %rcx /* Put the page_list in %rcx */	212	movq %rdi, %rcx /* Put the page_list in %rcx */
85	xorq %rdi, %rdi	213	xorq %rdi, %rdi
86	xorq %rsi, %rsi	214	xorq %rsi, %rsi
@@ -112,36 +240,27 @@ identity_mapped:
112	movq %rcx, %rsi /* For ever source page do a copy */	240	movq %rcx, %rsi /* For ever source page do a copy */
113	andq $0xfffffffffffff000, %rsi	241	andq $0xfffffffffffff000, %rsi
114		242
		243	movq %rdi, %rdx
		244	movq %rsi, %rax
		245
		246	movq %r10, %rdi
115	movq $512, %rcx	247	movq $512, %rcx
116	rep ; movsq	248	rep ; movsq
117	jmp 0b
118	3:
119
120	/* To be certain of avoiding problems with self-modifying code
121	* I need to execute a serializing instruction here.
122	* So I flush the TLB by reloading %cr3 here, it's handy,
123	* and not processor dependent.
124	*/
125	movq %cr3, %rax
126	movq %rax, %cr3
127		249
128	/* set all of the registers to known values */	250	movq %rax, %rdi
129	/* leave %rsp alone */	251	movq %rdx, %rsi
		252	movq $512, %rcx
		253	rep ; movsq
130		254
131	xorq %rax, %rax	255	movq %rdx, %rdi
132	xorq %rbx, %rbx	256	movq %r10, %rsi
133	xorq %rcx, %rcx	257	movq $512, %rcx
134	xorq %rdx, %rdx	258	rep ; movsq
135	xorq %rsi, %rsi
136	xorq %rdi, %rdi
137	xorq %rbp, %rbp
138	xorq %r8, %r8
139	xorq %r9, %r9
140	xorq %r10, %r9
141	xorq %r11, %r11
142	xorq %r12, %r12
143	xorq %r13, %r13
144	xorq %r14, %r14
145	xorq %r15, %r15
146		259
		260	lea PAGE_SIZE(%rax), %rsi
		261	jmp 0b
		262	3:
147	ret	263	ret
		264
		265	.globl kexec_control_code_size
		266	.set kexec_control_code_size, . - relocate_kernel