aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2013-09-03 13:46:26 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2013-09-03 13:46:26 -0400
commit1ccfd5eaf8f0135a0ce030728d1739e0eea4e3ce (patch)
tree95ae125e755f694ad80866f1540e03576702319a
parentea98af133f8dac1aa4ec093f69e1165a5db2d1ad (diff)
parent224593215525a79fe1acfffaafa528af9dc6f738 (diff)
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux
Pull first batch of s390 updates from Martin Schwidefsky: "The most interesting change is that Martin converted s390 to generic hardirqs. Which means that all current architectures have been converted and that CONFIG_GENERIC_HARDIRQS can be removed. Martin prepared a patch for that already (see genirq branch), but the best time to merge that is probably at the end of the merge window / begin of -rc1. Another patch converts s390 to software referenced bits instead of relying on the reference bit in the storage key. Therefore s390 doesn't use storage keys anymore, except for kvm. Besides that we have improvements, cleanups and fixes in PCI, DASD and all over the place." * 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: (32 commits) s390/pci: use virtual memory for iommu bitmap s390/cio: fix unlocked access of global bitmap s390/pci: update function handle after resume from hibernate s390/pci: try harder to modify a function s390/pci: split lpf s390/hibernate: add early resume function s390/pci: add recover sysfs knob s390/pci: use claim_resource s390/pci/hotplug: convert to be builtin only s390/mm: implement software referenced bits s390/dasd: fix statistics for recovered requests s390/tx: allow program interruption filtering in user space s390/pgtable: fix mprotect for single-threaded KVM guests s390/time: return with irqs disabled from psw_idle s390/kprobes: add support for compare and branch instructions s390/switch_to: fix save_access_regs() / restore_access_regs() s390/bitops: fix inline assembly constraints s390/dasd: enable raw_track_access reads without direct I/O s390/mm: introduce ptep_flush_lazy helper s390/time: clock comparator revalidation ...
Diffstat
-rw-r--r--arch/s390/Kconfig11
-rw-r--r--arch/s390/include/asm/airq.h67
-rw-r--r--arch/s390/include/asm/bitops.h12
-rw-r--r--arch/s390/include/asm/cio.h1
-rw-r--r--arch/s390/include/asm/hardirq.h5
-rw-r--r--arch/s390/include/asm/hugetlb.h135
-rw-r--r--arch/s390/include/asm/hw_irq.h17
-rw-r--r--arch/s390/include/asm/irq.h35
-rw-r--r--arch/s390/include/asm/mmu_context.h3
-rw-r--r--arch/s390/include/asm/page.h19
-rw-r--r--arch/s390/include/asm/pci.h54
-rw-r--r--arch/s390/include/asm/pci_insn.h12
-rw-r--r--arch/s390/include/asm/pci_io.h10
-rw-r--r--arch/s390/include/asm/pgtable.h637
-rw-r--r--arch/s390/include/asm/serial.h6
-rw-r--r--arch/s390/include/asm/switch_to.h9
-rw-r--r--arch/s390/include/asm/tlb.h3
-rw-r--r--arch/s390/include/asm/tlbflush.h6
-rw-r--r--arch/s390/kernel/entry.S16
-rw-r--r--arch/s390/kernel/entry64.S11
-rw-r--r--arch/s390/kernel/irq.c160
-rw-r--r--arch/s390/kernel/kprobes.c21
-rw-r--r--arch/s390/kernel/nmi.c5
-rw-r--r--arch/s390/kernel/process.c1
-rw-r--r--arch/s390/kernel/ptrace.c8
-rw-r--r--arch/s390/kernel/suspend.c11
-rw-r--r--arch/s390/kernel/swsusp_asm64.S7
-rw-r--r--arch/s390/kernel/time.c1
-rw-r--r--arch/s390/kernel/vdso.c6
-rw-r--r--arch/s390/lib/delay.c2
-rw-r--r--arch/s390/lib/uaccess_pt.c16
-rw-r--r--arch/s390/mm/dump_pagetables.c18
-rw-r--r--arch/s390/mm/gup.c6
-rw-r--r--arch/s390/mm/hugetlbpage.c124
-rw-r--r--arch/s390/mm/pageattr.c2
-rw-r--r--arch/s390/mm/pgtable.c83
-rw-r--r--arch/s390/mm/vmem.c15
-rw-r--r--arch/s390/pci/Makefile2
-rw-r--r--arch/s390/pci/pci.c575
-rw-r--r--arch/s390/pci/pci_clp.c146
-rw-r--r--arch/s390/pci/pci_dma.c16
-rw-r--r--arch/s390/pci/pci_event.c2
-rw-r--r--arch/s390/pci/pci_insn.c18
-rw-r--r--arch/s390/pci/pci_msi.c142
-rw-r--r--arch/s390/pci/pci_sysfs.c27
-rw-r--r--drivers/pci/hotplug/Kconfig2
-rw-r--r--drivers/pci/hotplug/s390_pci_hpc.c63
-rw-r--r--drivers/s390/block/dasd_devmap.c8
-rw-r--r--drivers/s390/block/dasd_eckd.c54
-rw-r--r--drivers/s390/block/dasd_erp.c14
-rw-r--r--drivers/s390/char/sclp_config.c2
-rw-r--r--drivers/s390/cio/airq.c174
-rw-r--r--drivers/s390/cio/ccwgroup.c2
-rw-r--r--drivers/s390/cio/cio.c46
-rw-r--r--drivers/s390/cio/cio.h3
-rw-r--r--drivers/s390/cio/cmf.c2
-rw-r--r--drivers/s390/cio/css.c4
-rw-r--r--drivers/s390/cio/css.h2
-rw-r--r--drivers/s390/cio/device.c2
-rw-r--r--drivers/s390/net/qeth_l3_sys.c2
-rw-r--r--include/asm-generic/pgtable.h4
-rw-r--r--mm/rmap.c3
62 files changed, 1468 insertions, 1402 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 8a4cae78f03c..8b7892bf6d8b 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -116,6 +116,7 @@ config S390
116 select HAVE_FUNCTION_GRAPH_TRACER 116 select HAVE_FUNCTION_GRAPH_TRACER
117 select HAVE_FUNCTION_TRACER 117 select HAVE_FUNCTION_TRACER
118 select HAVE_FUNCTION_TRACE_MCOUNT_TEST 118 select HAVE_FUNCTION_TRACE_MCOUNT_TEST
119 select HAVE_GENERIC_HARDIRQS
119 select HAVE_KERNEL_BZIP2 120 select HAVE_KERNEL_BZIP2
120 select HAVE_KERNEL_GZIP 121 select HAVE_KERNEL_GZIP
121 select HAVE_KERNEL_LZ4 122 select HAVE_KERNEL_LZ4
@@ -445,6 +446,16 @@ config PCI_NR_FUNCTIONS
445 This allows you to specify the maximum number of PCI functions which 446 This allows you to specify the maximum number of PCI functions which
446 this kernel will support. 447 this kernel will support.
447 448
449config PCI_NR_MSI
450 int "Maximum number of MSI interrupts (64-32768)"
451 range 64 32768
452 default "256"
453 help
454 This defines the number of virtual interrupts the kernel will
455 provide for MSI interrupts. If you configure your system to have
456 too few drivers will fail to allocate MSI interrupts for all
457 PCI devices.
458
448source "drivers/pci/Kconfig" 459source "drivers/pci/Kconfig"
449source "drivers/pci/pcie/Kconfig" 460source "drivers/pci/pcie/Kconfig"
450source "drivers/pci/hotplug/Kconfig" 461source "drivers/pci/hotplug/Kconfig"
diff --git a/arch/s390/include/asm/airq.h b/arch/s390/include/asm/airq.h
index 4066cee0c2d2..4bbb5957ed1b 100644
--- a/arch/s390/include/asm/airq.h
+++ b/arch/s390/include/asm/airq.h
@@ -9,6 +9,8 @@
9#ifndef _ASM_S390_AIRQ_H 9#ifndef _ASM_S390_AIRQ_H
10#define _ASM_S390_AIRQ_H 10#define _ASM_S390_AIRQ_H
11 11
12#include <linux/bit_spinlock.h>
13
12struct airq_struct { 14struct airq_struct {
13 struct hlist_node list; /* Handler queueing. */ 15 struct hlist_node list; /* Handler queueing. */
14 void (*handler)(struct airq_struct *); /* Thin-interrupt handler */ 16 void (*handler)(struct airq_struct *); /* Thin-interrupt handler */
@@ -23,4 +25,69 @@ struct airq_struct {
23int register_adapter_interrupt(struct airq_struct *airq); 25int register_adapter_interrupt(struct airq_struct *airq);
24void unregister_adapter_interrupt(struct airq_struct *airq); 26void unregister_adapter_interrupt(struct airq_struct *airq);
25 27
28/* Adapter interrupt bit vector */
29struct airq_iv {
30 unsigned long *vector; /* Adapter interrupt bit vector */
31 unsigned long *avail; /* Allocation bit mask for the bit vector */
32 unsigned long *bitlock; /* Lock bit mask for the bit vector */
33 unsigned long *ptr; /* Pointer associated with each bit */
34 unsigned int *data; /* 32 bit value associated with each bit */
35 unsigned long bits; /* Number of bits in the vector */
36 unsigned long end; /* Number of highest allocated bit + 1 */
37 spinlock_t lock; /* Lock to protect alloc & free */
38};
39
40#define AIRQ_IV_ALLOC 1 /* Use an allocation bit mask */
41#define AIRQ_IV_BITLOCK 2 /* Allocate the lock bit mask */
42#define AIRQ_IV_PTR 4 /* Allocate the ptr array */
43#define AIRQ_IV_DATA 8 /* Allocate the data array */
44
45struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags);
46void airq_iv_release(struct airq_iv *iv);
47unsigned long airq_iv_alloc_bit(struct airq_iv *iv);
48void airq_iv_free_bit(struct airq_iv *iv, unsigned long bit);
49unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,
50 unsigned long end);
51
52static inline unsigned long airq_iv_end(struct airq_iv *iv)
53{
54 return iv->end;
55}
56
57static inline void airq_iv_lock(struct airq_iv *iv, unsigned long bit)
58{
59 const unsigned long be_to_le = BITS_PER_LONG - 1;
60 bit_spin_lock(bit ^ be_to_le, iv->bitlock);
61}
62
63static inline void airq_iv_unlock(struct airq_iv *iv, unsigned long bit)
64{
65 const unsigned long be_to_le = BITS_PER_LONG - 1;
66 bit_spin_unlock(bit ^ be_to_le, iv->bitlock);
67}
68
69static inline void airq_iv_set_data(struct airq_iv *iv, unsigned long bit,
70 unsigned int data)
71{
72 iv->data[bit] = data;
73}
74
75static inline unsigned int airq_iv_get_data(struct airq_iv *iv,
76 unsigned long bit)
77{
78 return iv->data[bit];
79}
80
81static inline void airq_iv_set_ptr(struct airq_iv *iv, unsigned long bit,
82 unsigned long ptr)
83{
84 iv->ptr[bit] = ptr;
85}
86
87static inline unsigned long airq_iv_get_ptr(struct airq_iv *iv,
88 unsigned long bit)
89{
90 return iv->ptr[bit];
91}
92
26#endif /* _ASM_S390_AIRQ_H */ 93#endif /* _ASM_S390_AIRQ_H */
diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h
index 7d4676758733..10135a38673c 100644
--- a/arch/s390/include/asm/bitops.h
+++ b/arch/s390/include/asm/bitops.h
@@ -216,7 +216,7 @@ static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
216 addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3); 216 addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
217 asm volatile( 217 asm volatile(
218 " oc %O0(1,%R0),%1" 218 " oc %O0(1,%R0),%1"
219 : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc" ); 219 : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc");
220} 220}
221 221
222static inline void 222static inline void
@@ -244,7 +244,7 @@ __clear_bit(unsigned long nr, volatile unsigned long *ptr)
244 addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3); 244 addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
245 asm volatile( 245 asm volatile(
246 " nc %O0(1,%R0),%1" 246 " nc %O0(1,%R0),%1"
247 : "=Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7]) : "cc" ); 247 : "+Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7]) : "cc");
248} 248}
249 249
250static inline void 250static inline void
@@ -271,7 +271,7 @@ static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
271 addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3); 271 addr = (unsigned long) ptr + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
272 asm volatile( 272 asm volatile(
273 " xc %O0(1,%R0),%1" 273 " xc %O0(1,%R0),%1"
274 : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc" ); 274 : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) : "cc");
275} 275}
276 276
277static inline void 277static inline void
@@ -301,7 +301,7 @@ test_and_set_bit_simple(unsigned long nr, volatile unsigned long *ptr)
301 ch = *(unsigned char *) addr; 301 ch = *(unsigned char *) addr;
302 asm volatile( 302 asm volatile(
303 " oc %O0(1,%R0),%1" 303 " oc %O0(1,%R0),%1"
304 : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) 304 : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7])
305 : "cc", "memory"); 305 : "cc", "memory");
306 return (ch >> (nr & 7)) & 1; 306 return (ch >> (nr & 7)) & 1;
307} 307}
@@ -320,7 +320,7 @@ test_and_clear_bit_simple(unsigned long nr, volatile unsigned long *ptr)
320 ch = *(unsigned char *) addr; 320 ch = *(unsigned char *) addr;
321 asm volatile( 321 asm volatile(
322 " nc %O0(1,%R0),%1" 322 " nc %O0(1,%R0),%1"
323 : "=Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7]) 323 : "+Q" (*(char *) addr) : "Q" (_ni_bitmap[nr & 7])
324 : "cc", "memory"); 324 : "cc", "memory");
325 return (ch >> (nr & 7)) & 1; 325 return (ch >> (nr & 7)) & 1;
326} 326}
@@ -339,7 +339,7 @@ test_and_change_bit_simple(unsigned long nr, volatile unsigned long *ptr)
339 ch = *(unsigned char *) addr; 339 ch = *(unsigned char *) addr;
340 asm volatile( 340 asm volatile(
341 " xc %O0(1,%R0),%1" 341 " xc %O0(1,%R0),%1"
342 : "=Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7]) 342 : "+Q" (*(char *) addr) : "Q" (_oi_bitmap[nr & 7])
343 : "cc", "memory"); 343 : "cc", "memory");
344 return (ch >> (nr & 7)) & 1; 344 return (ch >> (nr & 7)) & 1;
345} 345}
diff --git a/arch/s390/include/asm/cio.h b/arch/s390/include/asm/cio.h
index ffb898961c8d..d42625053c37 100644
--- a/arch/s390/include/asm/cio.h
+++ b/arch/s390/include/asm/cio.h
@@ -296,6 +296,7 @@ static inline int ccw_dev_id_is_equal(struct ccw_dev_id *dev_id1,
296 return 0; 296 return 0;
297} 297}
298 298
299void channel_subsystem_reinit(void);
299extern void css_schedule_reprobe(void); 300extern void css_schedule_reprobe(void);
300 301
301extern void reipl_ccw_dev(struct ccw_dev_id *id); 302extern void reipl_ccw_dev(struct ccw_dev_id *id);
diff --git a/arch/s390/include/asm/hardirq.h b/arch/s390/include/asm/hardirq.h
index 0c82ba86e997..a908d2941c5d 100644
--- a/arch/s390/include/asm/hardirq.h
+++ b/arch/s390/include/asm/hardirq.h
@@ -20,4 +20,9 @@
20 20
21#define HARDIRQ_BITS 8 21#define HARDIRQ_BITS 8
22 22
23static inline void ack_bad_irq(unsigned int irq)
24{
25 printk(KERN_CRIT "unexpected IRQ trap at vector %02x\n", irq);
26}
27
23#endif /* __ASM_HARDIRQ_H */ 28#endif /* __ASM_HARDIRQ_H */
diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h
index bd90359d6d22..11eae5f55b70 100644
--- a/arch/s390/include/asm/hugetlb.h
+++ b/arch/s390/include/asm/hugetlb.h
@@ -17,6 +17,9 @@
17 17
18void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 18void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
19 pte_t *ptep, pte_t pte); 19 pte_t *ptep, pte_t pte);
20pte_t huge_ptep_get(pte_t *ptep);
21pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
22 unsigned long addr, pte_t *ptep);
20 23
21/* 24/*
22 * If the arch doesn't supply something else, assume that hugepage 25 * If the arch doesn't supply something else, assume that hugepage
@@ -38,147 +41,75 @@ static inline int prepare_hugepage_range(struct file *file,
38int arch_prepare_hugepage(struct page *page); 41int arch_prepare_hugepage(struct page *page);
39void arch_release_hugepage(struct page *page); 42void arch_release_hugepage(struct page *page);
40 43
41static inline pte_t huge_pte_wrprotect(pte_t pte) 44static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
45 pte_t *ptep)
42{ 46{
43 pte_val(pte) |= _PAGE_RO; 47 pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY;
44 return pte;
45} 48}
46 49
47static inline int huge_pte_none(pte_t pte) 50static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
51 unsigned long address, pte_t *ptep)
48{ 52{
49 return (pte_val(pte) & _SEGMENT_ENTRY_INV) && 53 huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
50 !(pte_val(pte) & _SEGMENT_ENTRY_RO);
51} 54}
52 55
53static inline pte_t huge_ptep_get(pte_t *ptep) 56static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
57 unsigned long addr, pte_t *ptep,
58 pte_t pte, int dirty)
54{ 59{
55 pte_t pte = *ptep; 60 int changed = !pte_same(huge_ptep_get(ptep), pte);
56 unsigned long mask; 61 if (changed) {
57 62 huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
58 if (!MACHINE_HAS_HPAGE) { 63 set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
59 ptep = (pte_t *) (pte_val(pte) & _SEGMENT_ENTRY_ORIGIN);
60 if (ptep) {
61 mask = pte_val(pte) &
62 (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO);
63 pte = pte_mkhuge(*ptep);
64 pte_val(pte) |= mask;
65 }
66 } 64 }
67 return pte; 65 return changed;
68} 66}
69 67
70static inline void __pmd_csp(pmd_t *pmdp) 68static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
69 unsigned long addr, pte_t *ptep)
71{ 70{
72 register unsigned long reg2 asm("2") = pmd_val(*pmdp); 71 pte_t pte = huge_ptep_get_and_clear(mm, addr, ptep);
73 register unsigned long reg3 asm("3") = pmd_val(*pmdp) | 72 set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte));
74 _SEGMENT_ENTRY_INV;
75 register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5;
76
77 asm volatile(
78 " csp %1,%3"
79 : "=m" (*pmdp)
80 : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc");
81} 73}
82 74
83static inline void huge_ptep_invalidate(struct mm_struct *mm, 75static inline pte_t mk_huge_pte(struct page *page, pgprot_t pgprot)
84 unsigned long address, pte_t *ptep)
85{
86 pmd_t *pmdp = (pmd_t *) ptep;
87
88 if (MACHINE_HAS_IDTE)
89 __pmd_idte(address, pmdp);
90 else
91 __pmd_csp(pmdp);
92 pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
93}
94
95static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
96 unsigned long addr, pte_t *ptep)
97{
98 pte_t pte = huge_ptep_get(ptep);
99
100 huge_ptep_invalidate(mm, addr, ptep);
101 return pte;
102}
103
104#define huge_ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \
105({ \
106 int __changed = !pte_same(huge_ptep_get(__ptep), __entry); \
107 if (__changed) { \
108 huge_ptep_invalidate((__vma)->vm_mm, __addr, __ptep); \
109 set_huge_pte_at((__vma)->vm_mm, __addr, __ptep, __entry); \
110 } \
111 __changed; \
112})
113
114#define huge_ptep_set_wrprotect(__mm, __addr, __ptep) \
115({ \
116 pte_t __pte = huge_ptep_get(__ptep); \
117 if (huge_pte_write(__pte)) { \
118 huge_ptep_invalidate(__mm, __addr, __ptep); \
119 set_huge_pte_at(__mm, __addr, __ptep, \
120 huge_pte_wrprotect(__pte)); \
121 } \
122})
123
124static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
125 unsigned long address, pte_t *ptep)
126{ 76{
127 huge_ptep_invalidate(vma->vm_mm, address, ptep); 77 return mk_pte(page, pgprot);
128} 78}
129 79
130static inline pte_t mk_huge_pte(struct page *page, pgprot_t pgprot) 80static inline int huge_pte_none(pte_t pte)
131{ 81{
132 pte_t pte; 82 return pte_none(pte);
133 pmd_t pmd;
134
135 pmd = mk_pmd_phys(page_to_phys(page), pgprot);
136 pte_val(pte) = pmd_val(pmd);
137 return pte;
138} 83}
139 84
140static inline int huge_pte_write(pte_t pte) 85static inline int huge_pte_write(pte_t pte)
141{ 86{
142 pmd_t pmd; 87 return pte_write(pte);
143
144 pmd_val(pmd) = pte_val(pte);
145 return pmd_write(pmd);
146} 88}
147 89
148static inline int huge_pte_dirty(pte_t pte) 90static inline int huge_pte_dirty(pte_t pte)
149{ 91{
150 /* No dirty bit in the segment table entry. */ 92 return pte_dirty(pte);
151 return 0;
152} 93}
153 94
154static inline pte_t huge_pte_mkwrite(pte_t pte) 95static inline pte_t huge_pte_mkwrite(pte_t pte)
155{ 96{
156 pmd_t pmd; 97 return pte_mkwrite(pte);
157
158 pmd_val(pmd) = pte_val(pte);
159 pte_val(pte) = pmd_val(pmd_mkwrite(pmd));
160 return pte;
161} 98}
162 99
163static inline pte_t huge_pte_mkdirty(pte_t pte) 100static inline pte_t huge_pte_mkdirty(pte_t pte)
164{ 101{
165 /* No dirty bit in the segment table entry. */ 102 return pte_mkdirty(pte);
166 return pte;
167} 103}
168 104
169static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot) 105static inline pte_t huge_pte_wrprotect(pte_t pte)
170{ 106{
171 pmd_t pmd; 107 return pte_wrprotect(pte);
172
173 pmd_val(pmd) = pte_val(pte);
174 pte_val(pte) = pmd_val(pmd_modify(pmd, newprot));
175 return pte;
176} 108}
177 109
178static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr, 110static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot)
179 pte_t *ptep)
180{ 111{
181 pmd_clear((pmd_t *) ptep); 112 return pte_modify(pte, newprot);
182} 113}
183 114
184#endif /* _ASM_S390_HUGETLB_H */ 115#endif /* _ASM_S390_HUGETLB_H */
diff --git a/arch/s390/include/asm/hw_irq.h b/arch/s390/include/asm/hw_irq.h
index 7e3d2586c1ff..ee96a8b697f9 100644
--- a/arch/s390/include/asm/hw_irq.h
+++ b/arch/s390/include/asm/hw_irq.h
@@ -4,19 +4,8 @@
4#include <linux/msi.h> 4#include <linux/msi.h>
5#include <linux/pci.h> 5#include <linux/pci.h>
6 6
7static inline struct msi_desc *irq_get_msi_desc(unsigned int irq) 7void __init init_airq_interrupts(void);
8{ 8void __init init_cio_interrupts(void);
9 return __irq_get_msi_desc(irq); 9void __init init_ext_interrupts(void);
10}
11
12/* Must be called with msi map lock held */
13static inline int irq_set_msi_desc(unsigned int irq, struct msi_desc *msi)
14{
15 if (!msi)
16 return -EINVAL;
17
18 msi->irq = irq;
19 return 0;
20}
21 10
22#endif 11#endif
diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h
index 87c17bfb2968..1eaa3625803c 100644
--- a/arch/s390/include/asm/irq.h
+++ b/arch/s390/include/asm/irq.h
@@ -1,17 +1,28 @@
1#ifndef _ASM_IRQ_H 1#ifndef _ASM_IRQ_H
2#define _ASM_IRQ_H 2#define _ASM_IRQ_H
3 3
4#define EXT_INTERRUPT 1
5#define IO_INTERRUPT 2
6#define THIN_INTERRUPT 3
7
8#define NR_IRQS_BASE 4
9
10#ifdef CONFIG_PCI_NR_MSI
11# define NR_IRQS (NR_IRQS_BASE + CONFIG_PCI_NR_MSI)
12#else
13# define NR_IRQS NR_IRQS_BASE
14#endif
15
16/* This number is used when no interrupt has been assigned */
17#define NO_IRQ 0
18
19#ifndef __ASSEMBLY__
20
4#include <linux/hardirq.h> 21#include <linux/hardirq.h>
5#include <linux/percpu.h> 22#include <linux/percpu.h>
6#include <linux/cache.h> 23#include <linux/cache.h>
7#include <linux/types.h> 24#include <linux/types.h>
8 25
9enum interruption_main_class {
10 EXTERNAL_INTERRUPT,
11 IO_INTERRUPT,
12 NR_IRQS
13};
14
15enum interruption_class { 26enum interruption_class {
16 IRQEXT_CLK, 27 IRQEXT_CLK,
17 IRQEXT_EXC, 28 IRQEXT_EXC,
@@ -72,14 +83,8 @@ void service_subclass_irq_unregister(void);
72void measurement_alert_subclass_register(void); 83void measurement_alert_subclass_register(void);
73void measurement_alert_subclass_unregister(void); 84void measurement_alert_subclass_unregister(void);
74 85
75#ifdef CONFIG_LOCKDEP 86#define irq_canonicalize(irq) (irq)
76# define disable_irq_nosync_lockdep(irq) disable_irq_nosync(irq) 87
77# define disable_irq_nosync_lockdep_irqsave(irq, flags) \ 88#endif /* __ASSEMBLY__ */
78 disable_irq_nosync(irq)
79# define disable_irq_lockdep(irq) disable_irq(irq)
80# define enable_irq_lockdep(irq) enable_irq(irq)
81# define enable_irq_lockdep_irqrestore(irq, flags) \
82 enable_irq(irq)
83#endif
84 89
85#endif /* _ASM_IRQ_H */ 90#endif /* _ASM_IRQ_H */
diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index 084e7755ed9b..7b7fce4e8469 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -77,8 +77,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
77 WARN_ON(atomic_read(&prev->context.attach_count) < 0); 77 WARN_ON(atomic_read(&prev->context.attach_count) < 0);
78 atomic_inc(&next->context.attach_count); 78 atomic_inc(&next->context.attach_count);
79 /* Check for TLBs not flushed yet */ 79 /* Check for TLBs not flushed yet */
80 if (next->context.flush_mm) 80 __tlb_flush_mm_lazy(next);
81 __tlb_flush_mm(next);
82} 81}
83 82
84#define enter_lazy_tlb(mm,tsk) do { } while (0) 83#define enter_lazy_tlb(mm,tsk) do { } while (0)
diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h
index 5d64fb7619cc..1e51f2915b2e 100644
--- a/arch/s390/include/asm/page.h
+++ b/arch/s390/include/asm/page.h
@@ -32,16 +32,6 @@
32 32
33void storage_key_init_range(unsigned long start, unsigned long end); 33void storage_key_init_range(unsigned long start, unsigned long end);
34 34
35static inline unsigned long pfmf(unsigned long function, unsigned long address)
36{
37 asm volatile(
38 " .insn rre,0xb9af0000,%[function],%[address]"
39 : [address] "+a" (address)
40 : [function] "d" (function)
41 : "memory");
42 return address;
43}
44
45static inline void clear_page(void *page) 35static inline void clear_page(void *page)
46{ 36{
47 register unsigned long reg1 asm ("1") = 0; 37 register unsigned long reg1 asm ("1") = 0;
@@ -150,15 +140,6 @@ static inline int page_reset_referenced(unsigned long addr)
150#define _PAGE_FP_BIT 0x08 /* HW fetch protection bit */ 140#define _PAGE_FP_BIT 0x08 /* HW fetch protection bit */
151#define _PAGE_ACC_BITS 0xf0 /* HW access control bits */ 141#define _PAGE_ACC_BITS 0xf0 /* HW access control bits */
152 142
153/*
154 * Test and clear referenced bit in storage key.
155 */
156#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG
157static inline int page_test_and_clear_young(unsigned long pfn)
158{
159 return page_reset_referenced(pfn << PAGE_SHIFT);
160}
161
162struct page; 143struct page;
163void arch_free_page(struct page *page, int order); 144void arch_free_page(struct page *page, int order);
164void arch_alloc_page(struct page *page, int order); 145void arch_alloc_page(struct page *page, int order);
diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index 6e577ba0e5da..c290f13d1c47 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -6,6 +6,7 @@
6/* must be set before including pci_clp.h */ 6/* must be set before including pci_clp.h */
7#define PCI_BAR_COUNT 6 7#define PCI_BAR_COUNT 6
8 8
9#include <linux/pci.h>
9#include <asm-generic/pci.h> 10#include <asm-generic/pci.h>
10#include <asm-generic/pci-dma-compat.h> 11#include <asm-generic/pci-dma-compat.h>
11#include <asm/pci_clp.h> 12#include <asm/pci_clp.h>
@@ -53,14 +54,9 @@ struct zpci_fmb {
53 atomic64_t unmapped_pages; 54 atomic64_t unmapped_pages;
54} __packed __aligned(16); 55} __packed __aligned(16);
55 56
56struct msi_map { 57#define ZPCI_MSI_VEC_BITS 11
57 unsigned long irq; 58#define ZPCI_MSI_VEC_MAX (1 << ZPCI_MSI_VEC_BITS)
58 struct msi_desc *msi; 59#define ZPCI_MSI_VEC_MASK (ZPCI_MSI_VEC_MAX - 1)
59 struct hlist_node msi_chain;
60};
61
62#define ZPCI_NR_MSI_VECS 64
63#define ZPCI_MSI_MASK (ZPCI_NR_MSI_VECS - 1)
64 60
65enum zpci_state { 61enum zpci_state {
66 ZPCI_FN_STATE_RESERVED, 62 ZPCI_FN_STATE_RESERVED,
@@ -91,8 +87,7 @@ struct zpci_dev {
91 87
92 /* IRQ stuff */ 88 /* IRQ stuff */
93 u64 msi_addr; /* MSI address */ 89 u64 msi_addr; /* MSI address */
94 struct zdev_irq_map *irq_map; 90 struct airq_iv *aibv; /* adapter interrupt bit vector */
95 struct msi_map *msi_map[ZPCI_NR_MSI_VECS];
96 unsigned int aisb; /* number of the summary bit */ 91 unsigned int aisb; /* number of the summary bit */
97 92
98 /* DMA stuff */ 93 /* DMA stuff */
@@ -122,11 +117,6 @@ struct zpci_dev {
122 struct dentry *debugfs_perf; 117 struct dentry *debugfs_perf;
123}; 118};
124 119
125struct pci_hp_callback_ops {
126 int (*create_slot) (struct zpci_dev *zdev);
127 void (*remove_slot) (struct zpci_dev *zdev);
128};
129
130static inline bool zdev_enabled(struct zpci_dev *zdev) 120static inline bool zdev_enabled(struct zpci_dev *zdev)
131{ 121{
132 return (zdev->fh & (1UL << 31)) ? true : false; 122 return (zdev->fh & (1UL << 31)) ? true : false;
@@ -146,32 +136,38 @@ int zpci_register_ioat(struct zpci_dev *, u8, u64, u64, u64);
146int zpci_unregister_ioat(struct zpci_dev *, u8); 136int zpci_unregister_ioat(struct zpci_dev *, u8);
147 137
148/* CLP */ 138/* CLP */
149int clp_find_pci_devices(void); 139int clp_scan_pci_devices(void);
140int clp_rescan_pci_devices(void);
141int clp_rescan_pci_devices_simple(void);
150int clp_add_pci_device(u32, u32, int); 142int clp_add_pci_device(u32, u32, int);
151int clp_enable_fh(struct zpci_dev *, u8); 143int clp_enable_fh(struct zpci_dev *, u8);
152int clp_disable_fh(struct zpci_dev *); 144int clp_disable_fh(struct zpci_dev *);
153 145
154/* MSI */
155struct msi_desc *__irq_get_msi_desc(unsigned int);
156int zpci_msi_set_mask_bits(struct msi_desc *, u32, u32);
157int zpci_setup_msi_irq(struct zpci_dev *, struct msi_desc *, unsigned int, int);
158void zpci_teardown_msi_irq(struct zpci_dev *, struct msi_desc *);
159int zpci_msihash_init(void);
160void zpci_msihash_exit(void);
161
162#ifdef CONFIG_PCI 146#ifdef CONFIG_PCI
163/* Error handling and recovery */ 147/* Error handling and recovery */
164void zpci_event_error(void *); 148void zpci_event_error(void *);
165void zpci_event_availability(void *); 149void zpci_event_availability(void *);
150void zpci_rescan(void);
166#else /* CONFIG_PCI */ 151#else /* CONFIG_PCI */
167static inline void zpci_event_error(void *e) {} 152static inline void zpci_event_error(void *e) {}
168static inline void zpci_event_availability(void *e) {} 153static inline void zpci_event_availability(void *e) {}
154static inline void zpci_rescan(void) {}
169#endif /* CONFIG_PCI */ 155#endif /* CONFIG_PCI */
170 156
157#ifdef CONFIG_HOTPLUG_PCI_S390
158int zpci_init_slot(struct zpci_dev *);
159void zpci_exit_slot(struct zpci_dev *);
160#else /* CONFIG_HOTPLUG_PCI_S390 */
161static inline int zpci_init_slot(struct zpci_dev *zdev)
162{
163 return 0;
164}
165static inline void zpci_exit_slot(struct zpci_dev *zdev) {}
166#endif /* CONFIG_HOTPLUG_PCI_S390 */
167
171/* Helpers */ 168/* Helpers */
172struct zpci_dev *get_zdev(struct pci_dev *); 169struct zpci_dev *get_zdev(struct pci_dev *);
173struct zpci_dev *get_zdev_by_fid(u32); 170struct zpci_dev *get_zdev_by_fid(u32);
174bool zpci_fid_present(u32);
175 171
176/* sysfs */ 172/* sysfs */
177int zpci_sysfs_add_device(struct device *); 173int zpci_sysfs_add_device(struct device *);
@@ -181,14 +177,6 @@ void zpci_sysfs_remove_device(struct device *);
181int zpci_dma_init(void); 177int zpci_dma_init(void);
182void zpci_dma_exit(void); 178void zpci_dma_exit(void);
183 179
184/* Hotplug */
185extern struct mutex zpci_list_lock;
186extern struct list_head zpci_list;
187extern unsigned int s390_pci_probe;
188
189void zpci_register_hp_ops(struct pci_hp_callback_ops *);
190void zpci_deregister_hp_ops(void);
191
192/* FMB */ 180/* FMB */
193int zpci_fmb_enable_device(struct zpci_dev *); 181int zpci_fmb_enable_device(struct zpci_dev *);
194int zpci_fmb_disable_device(struct zpci_dev *); 182int zpci_fmb_disable_device(struct zpci_dev *);
diff --git a/arch/s390/include/asm/pci_insn.h b/arch/s390/include/asm/pci_insn.h
index e6a2bdd4d705..df6eac9f0cb4 100644
--- a/arch/s390/include/asm/pci_insn.h
+++ b/arch/s390/include/asm/pci_insn.h
@@ -79,11 +79,11 @@ struct zpci_fib {
79} __packed; 79} __packed;
80 80
81 81
82int s390pci_mod_fc(u64 req, struct zpci_fib *fib); 82int zpci_mod_fc(u64 req, struct zpci_fib *fib);
83int s390pci_refresh_trans(u64 fn, u64 addr, u64 range); 83int zpci_refresh_trans(u64 fn, u64 addr, u64 range);
84int s390pci_load(u64 *data, u64 req, u64 offset); 84int zpci_load(u64 *data, u64 req, u64 offset);
85int s390pci_store(u64 data, u64 req, u64 offset); 85int zpci_store(u64 data, u64 req, u64 offset);
86int s390pci_store_block(const u64 *data, u64 req, u64 offset); 86int zpci_store_block(const u64 *data, u64 req, u64 offset);
87void set_irq_ctrl(u16 ctl, char *unused, u8 isc); 87void zpci_set_irq_ctrl(u16 ctl, char *unused, u8 isc);
88 88
89#endif 89#endif
diff --git a/arch/s390/include/asm/pci_io.h b/arch/s390/include/asm/pci_io.h
index 83a9caa6ae53..d194d544d694 100644
--- a/arch/s390/include/asm/pci_io.h
+++ b/arch/s390/include/asm/pci_io.h
@@ -36,7 +36,7 @@ static inline RETTYPE zpci_read_##RETTYPE(const volatile void __iomem *addr) \
36 u64 data; \ 36 u64 data; \
37 int rc; \ 37 int rc; \
38 \ 38 \
39 rc = s390pci_load(&data, req, ZPCI_OFFSET(addr)); \ 39 rc = zpci_load(&data, req, ZPCI_OFFSET(addr)); \
40 if (rc) \ 40 if (rc) \
41 data = -1ULL; \ 41 data = -1ULL; \
42 return (RETTYPE) data; \ 42 return (RETTYPE) data; \
@@ -50,7 +50,7 @@ static inline void zpci_write_##VALTYPE(VALTYPE val, \
50 u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, LENGTH); \ 50 u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, LENGTH); \
51 u64 data = (VALTYPE) val; \ 51 u64 data = (VALTYPE) val; \
52 \ 52 \
53 s390pci_store(data, req, ZPCI_OFFSET(addr)); \ 53 zpci_store(data, req, ZPCI_OFFSET(addr)); \
54} 54}
55 55
56zpci_read(8, u64) 56zpci_read(8, u64)
@@ -83,7 +83,7 @@ static inline int zpci_write_single(u64 req, const u64 *data, u64 offset, u8 len
83 val = 0; /* let FW report error */ 83 val = 0; /* let FW report error */
84 break; 84 break;
85 } 85 }
86 return s390pci_store(val, req, offset); 86 return zpci_store(val, req, offset);
87} 87}
88 88
89static inline int zpci_read_single(u64 req, u64 *dst, u64 offset, u8 len) 89static inline int zpci_read_single(u64 req, u64 *dst, u64 offset, u8 len)
@@ -91,7 +91,7 @@ static inline int zpci_read_single(u64 req, u64 *dst, u64 offset, u8 len)
91 u64 data; 91 u64 data;
92 int cc; 92 int cc;
93 93
94 cc = s390pci_load(&data, req, offset); 94 cc = zpci_load(&data, req, offset);
95 if (cc) 95 if (cc)
96 goto out; 96 goto out;
97 97
@@ -115,7 +115,7 @@ out:
115 115
116static inline int zpci_write_block(u64 req, const u64 *data, u64 offset) 116static inline int zpci_write_block(u64 req, const u64 *data, u64 offset)
117{ 117{
118 return s390pci_store_block(data, req, offset); 118 return zpci_store_block(data, req, offset);
119} 119}
120 120
121static inline u8 zpci_get_max_write_size(u64 src, u64 dst, int len, int max) 121static inline u8 zpci_get_max_write_size(u64 src, u64 dst, int len, int max)
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 75fb726de91f..9f215b40109e 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -217,63 +217,57 @@ extern unsigned long MODULES_END;
217 217
218/* Hardware bits in the page table entry */ 218/* Hardware bits in the page table entry */
219#define _PAGE_CO 0x100 /* HW Change-bit override */ 219#define _PAGE_CO 0x100 /* HW Change-bit override */
220#define _PAGE_RO 0x200 /* HW read-only bit */ 220#define _PAGE_PROTECT 0x200 /* HW read-only bit */
221#define _PAGE_INVALID 0x400 /* HW invalid bit */ 221#define _PAGE_INVALID 0x400 /* HW invalid bit */
222#define _PAGE_LARGE 0x800 /* Bit to mark a large pte */
222 223
223/* Software bits in the page table entry */ 224/* Software bits in the page table entry */
224#define _PAGE_SWT 0x001 /* SW pte type bit t */ 225#define _PAGE_PRESENT 0x001 /* SW pte present bit */
225#define _PAGE_SWX 0x002 /* SW pte type bit x */ 226#define _PAGE_TYPE 0x002 /* SW pte type bit */
226#define _PAGE_SWC 0x004 /* SW pte changed bit */ 227#define _PAGE_YOUNG 0x004 /* SW pte young bit */
227#define _PAGE_SWR 0x008 /* SW pte referenced bit */ 228#define _PAGE_DIRTY 0x008 /* SW pte dirty bit */
228#define _PAGE_SWW 0x010 /* SW pte write bit */ 229#define _PAGE_READ 0x010 /* SW pte read bit */
229#define _PAGE_SPECIAL 0x020 /* SW associated with special page */ 230#define _PAGE_WRITE 0x020 /* SW pte write bit */
231#define _PAGE_SPECIAL 0x040 /* SW associated with special page */
230#define __HAVE_ARCH_PTE_SPECIAL 232#define __HAVE_ARCH_PTE_SPECIAL
231 233
232/* Set of bits not changed in pte_modify */ 234/* Set of bits not changed in pte_modify */
233#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_CO | \ 235#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_CO | \
234 _PAGE_SWC | _PAGE_SWR) 236 _PAGE_DIRTY | _PAGE_YOUNG)
235
236/* Six different types of pages. */
237#define _PAGE_TYPE_EMPTY 0x400
238#define _PAGE_TYPE_NONE 0x401
239#define _PAGE_TYPE_SWAP 0x403
240#define _PAGE_TYPE_FILE 0x601 /* bit 0x002 is used for offset !! */
241#define _PAGE_TYPE_RO 0x200
242#define _PAGE_TYPE_RW 0x000
243 237
244/* 238/*
245 * Only four types for huge pages, using the invalid bit and protection bit 239 * handle_pte_fault uses pte_present, pte_none and pte_file to find out the
246 * of a segment table entry. 240 * pte type WITHOUT holding the page table lock. The _PAGE_PRESENT bit
247 */ 241 * is used to distinguish present from not-present ptes. It is changed only
248#define _HPAGE_TYPE_EMPTY 0x020 /* _SEGMENT_ENTRY_INV */ 242 * with the page table lock held.
249#define _HPAGE_TYPE_NONE 0x220 243 *
250#define _HPAGE_TYPE_RO 0x200 /* _SEGMENT_ENTRY_RO */ 244 * The following table gives the different possible bit combinations for
251#define _HPAGE_TYPE_RW 0x000 245 * the pte hardware and software bits in the last 12 bits of a pte:
252
253/*
254 * PTE type bits are rather complicated. handle_pte_fault uses pte_present,
255 * pte_none and pte_file to find out the pte type WITHOUT holding the page
256 * table lock. ptep_clear_flush on the other hand uses ptep_clear_flush to
257 * invalidate a given pte. ipte sets the hw invalid bit and clears all tlbs
258 * for the page. The page table entry is set to _PAGE_TYPE_EMPTY afterwards.
259 * This change is done while holding the lock, but the intermediate step
260 * of a previously valid pte with the hw invalid bit set can be observed by
261 * handle_pte_fault. That makes it necessary that all valid pte types with
262 * the hw invalid bit set must be distinguishable from the four pte types
263 * empty, none, swap and file.
264 * 246 *
265 * irxt ipte irxt 247 * 842100000000
266 * _PAGE_TYPE_EMPTY 1000 -> 1000 248 * 000084210000
267 * _PAGE_TYPE_NONE 1001 -> 1001 249 * 000000008421
268 * _PAGE_TYPE_SWAP 1011 -> 1011 250 * .IR...wrdytp
269 * _PAGE_TYPE_FILE 11?1 -> 11?1 251 * empty .10...000000
270 * _PAGE_TYPE_RO 0100 -> 1100 252 * swap .10...xxxx10
271 * _PAGE_TYPE_RW 0000 -> 1000 253 * file .11...xxxxx0
254 * prot-none, clean, old .11...000001
255 * prot-none, clean, young .11...000101
256 * prot-none, dirty, old .10...001001
257 * prot-none, dirty, young .10...001101
258 * read-only, clean, old .11...010001
259 * read-only, clean, young .01...010101
260 * read-only, dirty, old .11...011001
261 * read-only, dirty, young .01...011101
262 * read-write, clean, old .11...110001
263 * read-write, clean, young .01...110101
264 * read-write, dirty, old .10...111001
265 * read-write, dirty, young .00...111101
272 * 266 *
273 * pte_none is true for bits combinations 1000, 1010, 1100, 1110 267 * pte_present is true for the bit pattern .xx...xxxxx1, (pte & 0x001) == 0x001
274 * pte_present is true for bits combinations 0000, 0010, 0100, 0110, 1001 268 * pte_none is true for the bit pattern .10...xxxx00, (pte & 0x603) == 0x400
275 * pte_file is true for bits combinations 1101, 1111 269 * pte_file is true for the bit pattern .11...xxxxx0, (pte & 0x601) == 0x600
276 * swap pte is 1011 and 0001, 0011, 0101, 0111 are invalid. 270 * pte_swap is true for the bit pattern .10...xxxx10, (pte & 0x603) == 0x402
277 */ 271 */
278 272
279#ifndef CONFIG_64BIT 273#ifndef CONFIG_64BIT
@@ -286,14 +280,25 @@ extern unsigned long MODULES_END;
286#define _ASCE_TABLE_LENGTH 0x7f /* 128 x 64 entries = 8k */ 280#define _ASCE_TABLE_LENGTH 0x7f /* 128 x 64 entries = 8k */
287 281
288/* Bits in the segment table entry */ 282/* Bits in the segment table entry */
283#define _SEGMENT_ENTRY_BITS 0x7fffffffUL /* Valid segment table bits */
289#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL /* page table origin */ 284#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL /* page table origin */
290#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */ 285#define _SEGMENT_ENTRY_PROTECT 0x200 /* page protection bit */
291#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */ 286#define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */
292#define _SEGMENT_ENTRY_COMMON 0x10 /* common segment bit */ 287#define _SEGMENT_ENTRY_COMMON 0x10 /* common segment bit */
293#define _SEGMENT_ENTRY_PTL 0x0f /* page table length */ 288#define _SEGMENT_ENTRY_PTL 0x0f /* page table length */
289#define _SEGMENT_ENTRY_NONE _SEGMENT_ENTRY_PROTECT
294 290
295#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL) 291#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL)
296#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV) 292#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID)
293
294/*
295 * Segment table entry encoding (I = invalid, R = read-only bit):
296 * ..R...I.....
297 * prot-none ..1...1.....
298 * read-only ..1...0.....
299 * read-write ..0...0.....
300 * empty ..0...1.....
301 */
297 302
298/* Page status table bits for virtualization */ 303/* Page status table bits for virtualization */
299#define PGSTE_ACC_BITS 0xf0000000UL 304#define PGSTE_ACC_BITS 0xf0000000UL
@@ -303,9 +308,7 @@ extern unsigned long MODULES_END;
303#define PGSTE_HC_BIT 0x00200000UL 308#define PGSTE_HC_BIT 0x00200000UL
304#define PGSTE_GR_BIT 0x00040000UL 309#define PGSTE_GR_BIT 0x00040000UL
305#define PGSTE_GC_BIT 0x00020000UL 310#define PGSTE_GC_BIT 0x00020000UL
306#define PGSTE_UR_BIT 0x00008000UL 311#define PGSTE_IN_BIT 0x00008000UL /* IPTE notify bit */
307#define PGSTE_UC_BIT 0x00004000UL /* user dirty (migration) */
308#define PGSTE_IN_BIT 0x00002000UL /* IPTE notify bit */
309 312
310#else /* CONFIG_64BIT */ 313#else /* CONFIG_64BIT */
311 314
@@ -324,8 +327,8 @@ extern unsigned long MODULES_END;
324 327
325/* Bits in the region table entry */ 328/* Bits in the region table entry */
326#define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */ 329#define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */
327#define _REGION_ENTRY_RO 0x200 /* region protection bit */ 330#define _REGION_ENTRY_PROTECT 0x200 /* region protection bit */
328#define _REGION_ENTRY_INV 0x20 /* invalid region table entry */ 331#define _REGION_ENTRY_INVALID 0x20 /* invalid region table entry */
329#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */ 332#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */
330#define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */ 333#define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */
331#define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */ 334#define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */
@@ -333,29 +336,47 @@ extern unsigned long MODULES_END;
333#define _REGION_ENTRY_LENGTH 0x03 /* region third length */ 336#define _REGION_ENTRY_LENGTH 0x03 /* region third length */
334 337
335#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH) 338#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
336#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV) 339#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID)
337#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH) 340#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
338#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV) 341#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID)
339#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH) 342#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
340#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV) 343#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID)
341 344
342#define _REGION3_ENTRY_LARGE 0x400 /* RTTE-format control, large page */ 345#define _REGION3_ENTRY_LARGE 0x400 /* RTTE-format control, large page */
343#define _REGION3_ENTRY_RO 0x200 /* page protection bit */ 346#define _REGION3_ENTRY_RO 0x200 /* page protection bit */
344#define _REGION3_ENTRY_CO 0x100 /* change-recording override */ 347#define _REGION3_ENTRY_CO 0x100 /* change-recording override */
345 348
346/* Bits in the segment table entry */ 349/* Bits in the segment table entry */
350#define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL
351#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff1ff33UL
347#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */ 352#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */
348#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */ 353#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */
349#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */ 354#define _SEGMENT_ENTRY_PROTECT 0x200 /* page protection bit */
350#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */ 355#define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */
351 356
352#define _SEGMENT_ENTRY (0) 357#define _SEGMENT_ENTRY (0)
353#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV) 358#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID)
354 359
355#define _SEGMENT_ENTRY_LARGE 0x400 /* STE-format control, large page */ 360#define _SEGMENT_ENTRY_LARGE 0x400 /* STE-format control, large page */
356#define _SEGMENT_ENTRY_CO 0x100 /* change-recording override */ 361#define _SEGMENT_ENTRY_CO 0x100 /* change-recording override */
362#define _SEGMENT_ENTRY_SPLIT 0x001 /* THP splitting bit */
363#define _SEGMENT_ENTRY_YOUNG 0x002 /* SW segment young bit */
364#define _SEGMENT_ENTRY_NONE _SEGMENT_ENTRY_YOUNG
365
366/*
367 * Segment table entry encoding (R = read-only, I = invalid, y = young bit):
368 * ..R...I...y.
369 * prot-none, old ..0...1...1.
370 * prot-none, young ..1...1...1.
371 * read-only, old ..1...1...0.
372 * read-only, young ..1...0...1.
373 * read-write, old ..0...1...0.
374 * read-write, young ..0...0...1.
375 * The segment table origin is used to distinguish empty (origin==0) from
376 * read-write, old segment table entries (origin!=0)
377 */
378
357#define _SEGMENT_ENTRY_SPLIT_BIT 0 /* THP splitting bit number */ 379#define _SEGMENT_ENTRY_SPLIT_BIT 0 /* THP splitting bit number */
358#define _SEGMENT_ENTRY_SPLIT (1UL << _SEGMENT_ENTRY_SPLIT_BIT)
359 380
360/* Set of bits not changed in pmd_modify */ 381/* Set of bits not changed in pmd_modify */
361#define _SEGMENT_CHG_MASK (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \ 382#define _SEGMENT_CHG_MASK (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \
@@ -369,9 +390,7 @@ extern unsigned long MODULES_END;
369#define PGSTE_HC_BIT 0x0020000000000000UL 390#define PGSTE_HC_BIT 0x0020000000000000UL
370#define PGSTE_GR_BIT 0x0004000000000000UL 391#define PGSTE_GR_BIT 0x0004000000000000UL
371#define PGSTE_GC_BIT 0x0002000000000000UL 392#define PGSTE_GC_BIT 0x0002000000000000UL
372#define PGSTE_UR_BIT 0x0000800000000000UL 393#define PGSTE_IN_BIT 0x0000800000000000UL /* IPTE notify bit */
373#define PGSTE_UC_BIT 0x0000400000000000UL /* user dirty (migration) */
374#define PGSTE_IN_BIT 0x0000200000000000UL /* IPTE notify bit */
375 394
376#endif /* CONFIG_64BIT */ 395#endif /* CONFIG_64BIT */
377 396
@@ -386,14 +405,18 @@ extern unsigned long MODULES_END;
386/* 405/*
387 * Page protection definitions. 406 * Page protection definitions.
388 */ 407 */
389#define PAGE_NONE __pgprot(_PAGE_TYPE_NONE) 408#define PAGE_NONE __pgprot(_PAGE_PRESENT | _PAGE_INVALID)
390#define PAGE_RO __pgprot(_PAGE_TYPE_RO) 409#define PAGE_READ __pgprot(_PAGE_PRESENT | _PAGE_READ | \
391#define PAGE_RW __pgprot(_PAGE_TYPE_RO | _PAGE_SWW) 410 _PAGE_INVALID | _PAGE_PROTECT)
392#define PAGE_RWC __pgprot(_PAGE_TYPE_RW | _PAGE_SWW | _PAGE_SWC) 411#define PAGE_WRITE __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
393 412 _PAGE_INVALID | _PAGE_PROTECT)
394#define PAGE_KERNEL PAGE_RWC 413
395#define PAGE_SHARED PAGE_KERNEL 414#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
396#define PAGE_COPY PAGE_RO 415 _PAGE_YOUNG | _PAGE_DIRTY)
416#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
417 _PAGE_YOUNG | _PAGE_DIRTY)
418#define PAGE_KERNEL_RO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \
419 _PAGE_PROTECT)
397 420
398/* 421/*
399 * On s390 the page table entry has an invalid bit and a read-only bit. 422 * On s390 the page table entry has an invalid bit and a read-only bit.
@@ -402,35 +425,31 @@ extern unsigned long MODULES_END;
402 */ 425 */
403 /*xwr*/ 426 /*xwr*/
404#define __P000 PAGE_NONE 427#define __P000 PAGE_NONE
405#define __P001 PAGE_RO 428#define __P001 PAGE_READ
406#define __P010 PAGE_RO 429#define __P010 PAGE_READ
407#define __P011 PAGE_RO 430#define __P011 PAGE_READ
408#define __P100 PAGE_RO 431#define __P100 PAGE_READ
409#define __P101 PAGE_RO 432#define __P101 PAGE_READ
410#define __P110 PAGE_RO 433#define __P110 PAGE_READ
411#define __P111 PAGE_RO 434#define __P111 PAGE_READ
412 435
413#define __S000 PAGE_NONE 436#define __S000 PAGE_NONE
414#define __S001 PAGE_RO 437#define __S001 PAGE_READ
415#define __S010 PAGE_RW 438#define __S010 PAGE_WRITE
416#define __S011 PAGE_RW 439#define __S011 PAGE_WRITE
417#define __S100 PAGE_RO 440#define __S100 PAGE_READ
418#define __S101 PAGE_RO 441#define __S101 PAGE_READ
419#define __S110 PAGE_RW 442#define __S110 PAGE_WRITE
420#define __S111 PAGE_RW 443#define __S111 PAGE_WRITE
421 444
422/* 445/*
423 * Segment entry (large page) protection definitions. 446 * Segment entry (large page) protection definitions.
424 */ 447 */
425#define SEGMENT_NONE __pgprot(_HPAGE_TYPE_NONE) 448#define SEGMENT_NONE __pgprot(_SEGMENT_ENTRY_INVALID | \
426#define SEGMENT_RO __pgprot(_HPAGE_TYPE_RO) 449 _SEGMENT_ENTRY_NONE)
427#define SEGMENT_RW __pgprot(_HPAGE_TYPE_RW) 450#define SEGMENT_READ __pgprot(_SEGMENT_ENTRY_INVALID | \
428 451 _SEGMENT_ENTRY_PROTECT)
429static inline int mm_exclusive(struct mm_struct *mm) 452#define SEGMENT_WRITE __pgprot(_SEGMENT_ENTRY_INVALID)
430{
431 return likely(mm == current->active_mm &&
432 atomic_read(&mm->context.attach_count) <= 1);
433}
434 453
435static inline int mm_has_pgste(struct mm_struct *mm) 454static inline int mm_has_pgste(struct mm_struct *mm)
436{ 455{
@@ -467,7 +486,7 @@ static inline int pgd_none(pgd_t pgd)
467{ 486{
468 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) 487 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
469 return 0; 488 return 0;
470 return (pgd_val(pgd) & _REGION_ENTRY_INV) != 0UL; 489 return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL;
471} 490}
472 491
473static inline int pgd_bad(pgd_t pgd) 492static inline int pgd_bad(pgd_t pgd)
@@ -478,7 +497,7 @@ static inline int pgd_bad(pgd_t pgd)
478 * invalid for either table entry. 497 * invalid for either table entry.
479 */ 498 */
480 unsigned long mask = 499 unsigned long mask =
481 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV & 500 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID &
482 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH; 501 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
483 return (pgd_val(pgd) & mask) != 0; 502 return (pgd_val(pgd) & mask) != 0;
484} 503}
@@ -494,7 +513,7 @@ static inline int pud_none(pud_t pud)
494{ 513{
495 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) 514 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
496 return 0; 515 return 0;
497 return (pud_val(pud) & _REGION_ENTRY_INV) != 0UL; 516 return (pud_val(pud) & _REGION_ENTRY_INVALID) != 0UL;
498} 517}
499 518
500static inline int pud_large(pud_t pud) 519static inline int pud_large(pud_t pud)
@@ -512,7 +531,7 @@ static inline int pud_bad(pud_t pud)
512 * invalid for either table entry. 531 * invalid for either table entry.
513 */ 532 */
514 unsigned long mask = 533 unsigned long mask =
515 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV & 534 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID &
516 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH; 535 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
517 return (pud_val(pud) & mask) != 0; 536 return (pud_val(pud) & mask) != 0;
518} 537}
@@ -521,30 +540,36 @@ static inline int pud_bad(pud_t pud)
521 540
522static inline int pmd_present(pmd_t pmd) 541static inline int pmd_present(pmd_t pmd)
523{ 542{
524 unsigned long mask = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO; 543 return pmd_val(pmd) != _SEGMENT_ENTRY_INVALID;
525 return (pmd_val(pmd) & mask) == _HPAGE_TYPE_NONE ||
526 !(pmd_val(pmd) & _SEGMENT_ENTRY_INV);
527} 544}
528 545
529static inline int pmd_none(pmd_t pmd) 546static inline int pmd_none(pmd_t pmd)
530{ 547{
531 return (pmd_val(pmd) & _SEGMENT_ENTRY_INV) && 548 return pmd_val(pmd) == _SEGMENT_ENTRY_INVALID;
532 !(pmd_val(pmd) & _SEGMENT_ENTRY_RO);
533} 549}
534 550
535static inline int pmd_large(pmd_t pmd) 551static inline int pmd_large(pmd_t pmd)
536{ 552{
537#ifdef CONFIG_64BIT 553#ifdef CONFIG_64BIT
538 return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE); 554 return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0;
539#else 555#else
540 return 0; 556 return 0;
541#endif 557#endif
542} 558}
543 559
560static inline int pmd_prot_none(pmd_t pmd)
561{
562 return (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) &&
563 (pmd_val(pmd) & _SEGMENT_ENTRY_NONE);
564}
565
544static inline int pmd_bad(pmd_t pmd) 566static inline int pmd_bad(pmd_t pmd)
545{ 567{
546 unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV; 568#ifdef CONFIG_64BIT
547 return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY; 569 if (pmd_large(pmd))
570 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0;
571#endif
572 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
548} 573}
549 574
550#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH 575#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
@@ -563,31 +588,40 @@ extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
563#define __HAVE_ARCH_PMD_WRITE 588#define __HAVE_ARCH_PMD_WRITE
564static inline int pmd_write(pmd_t pmd) 589static inline int pmd_write(pmd_t pmd)
565{ 590{
566 return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0; 591 if (pmd_prot_none(pmd))
592 return 0;
593 return (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) == 0;
567} 594}
568 595
569static inline int pmd_young(pmd_t pmd) 596static inline int pmd_young(pmd_t pmd)
570{ 597{
571 return 0; 598 int young = 0;
599#ifdef CONFIG_64BIT
600 if (pmd_prot_none(pmd))
601 young = (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT) != 0;
602 else
603 young = (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0;
604#endif
605 return young;
572} 606}
573 607
574static inline int pte_none(pte_t pte) 608static inline int pte_present(pte_t pte)
575{ 609{
576 return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT); 610 /* Bit pattern: (pte & 0x001) == 0x001 */
611 return (pte_val(pte) & _PAGE_PRESENT) != 0;
577} 612}
578 613
579static inline int pte_present(pte_t pte) 614static inline int pte_none(pte_t pte)
580{ 615{
581 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT | _PAGE_SWX; 616 /* Bit pattern: pte == 0x400 */
582 return (pte_val(pte) & mask) == _PAGE_TYPE_NONE || 617 return pte_val(pte) == _PAGE_INVALID;
583 (!(pte_val(pte) & _PAGE_INVALID) &&
584 !(pte_val(pte) & _PAGE_SWT));
585} 618}
586 619
587static inline int pte_file(pte_t pte) 620static inline int pte_file(pte_t pte)
588{ 621{
589 unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT; 622 /* Bit pattern: (pte & 0x601) == 0x600 */
590 return (pte_val(pte) & mask) == _PAGE_TYPE_FILE; 623 return (pte_val(pte) & (_PAGE_INVALID | _PAGE_PROTECT | _PAGE_PRESENT))
624 == (_PAGE_INVALID | _PAGE_PROTECT);
591} 625}
592 626
593static inline int pte_special(pte_t pte) 627static inline int pte_special(pte_t pte)
@@ -634,6 +668,15 @@ static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
634#endif 668#endif
635} 669}
636 670
671static inline pgste_t pgste_get(pte_t *ptep)
672{
673 unsigned long pgste = 0;
674#ifdef CONFIG_PGSTE
675 pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
676#endif
677 return __pgste(pgste);
678}
679
637static inline void pgste_set(pte_t *ptep, pgste_t pgste) 680static inline void pgste_set(pte_t *ptep, pgste_t pgste)
638{ 681{
639#ifdef CONFIG_PGSTE 682#ifdef CONFIG_PGSTE
@@ -644,33 +687,28 @@ static inline void pgste_set(pte_t *ptep, pgste_t pgste)
644static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste) 687static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste)
645{ 688{
646#ifdef CONFIG_PGSTE 689#ifdef CONFIG_PGSTE
647 unsigned long address, bits; 690 unsigned long address, bits, skey;
648 unsigned char skey;
649 691
650 if (pte_val(*ptep) & _PAGE_INVALID) 692 if (pte_val(*ptep) & _PAGE_INVALID)
651 return pgste; 693 return pgste;
652 address = pte_val(*ptep) & PAGE_MASK; 694 address = pte_val(*ptep) & PAGE_MASK;
653 skey = page_get_storage_key(address); 695 skey = (unsigned long) page_get_storage_key(address);
654 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); 696 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
655 /* Clear page changed & referenced bit in the storage key */ 697 if (!(pgste_val(pgste) & PGSTE_HC_BIT) && (bits & _PAGE_CHANGED)) {
656 if (bits & _PAGE_CHANGED) 698 /* Transfer dirty + referenced bit to host bits in pgste */
699 pgste_val(pgste) |= bits << 52;
657 page_set_storage_key(address, skey ^ bits, 0); 700 page_set_storage_key(address, skey ^ bits, 0);
658 else if (bits) 701 } else if (!(pgste_val(pgste) & PGSTE_HR_BIT) &&
702 (bits & _PAGE_REFERENCED)) {
703 /* Transfer referenced bit to host bit in pgste */
704 pgste_val(pgste) |= PGSTE_HR_BIT;
659 page_reset_referenced(address); 705 page_reset_referenced(address);
706 }
660 /* Transfer page changed & referenced bit to guest bits in pgste */ 707 /* Transfer page changed & referenced bit to guest bits in pgste */
661 pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ 708 pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */
662 /* Get host changed & referenced bits from pgste */
663 bits |= (pgste_val(pgste) & (PGSTE_HR_BIT | PGSTE_HC_BIT)) >> 52;
664 /* Transfer page changed & referenced bit to kvm user bits */
665 pgste_val(pgste) |= bits << 45; /* PGSTE_UR_BIT & PGSTE_UC_BIT */
666 /* Clear relevant host bits in pgste. */
667 pgste_val(pgste) &= ~(PGSTE_HR_BIT | PGSTE_HC_BIT);
668 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
669 /* Copy page access key and fetch protection bit to pgste */ 709 /* Copy page access key and fetch protection bit to pgste */
670 pgste_val(pgste) |= 710 pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
671 (unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; 711 pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
672 /* Transfer referenced bit to pte */
673 pte_val(*ptep) |= (bits & _PAGE_REFERENCED) << 1;
674#endif 712#endif
675 return pgste; 713 return pgste;
676 714
@@ -679,24 +717,11 @@ static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste)
679static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste) 717static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste)
680{ 718{
681#ifdef CONFIG_PGSTE 719#ifdef CONFIG_PGSTE
682 int young;
683
684 if (pte_val(*ptep) & _PAGE_INVALID) 720 if (pte_val(*ptep) & _PAGE_INVALID)
685 return pgste; 721 return pgste;
686 /* Get referenced bit from storage key */ 722 /* Get referenced bit from storage key */
687 young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); 723 if (page_reset_referenced(pte_val(*ptep) & PAGE_MASK))
688 if (young) 724 pgste_val(pgste) |= PGSTE_HR_BIT | PGSTE_GR_BIT;
689 pgste_val(pgste) |= PGSTE_GR_BIT;
690 /* Get host referenced bit from pgste */
691 if (pgste_val(pgste) & PGSTE_HR_BIT) {
692 pgste_val(pgste) &= ~PGSTE_HR_BIT;
693 young = 1;
694 }
695 /* Transfer referenced bit to kvm user bits and pte */
696 if (young) {
697 pgste_val(pgste) |= PGSTE_UR_BIT;
698 pte_val(*ptep) |= _PAGE_SWR;
699 }
700#endif 725#endif
701 return pgste; 726 return pgste;
702} 727}
@@ -723,13 +748,13 @@ static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry)
723 748
724static inline void pgste_set_pte(pte_t *ptep, pte_t entry) 749static inline void pgste_set_pte(pte_t *ptep, pte_t entry)
725{ 750{
726 if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_SWW)) { 751 if (!MACHINE_HAS_ESOP && (pte_val(entry) & _PAGE_WRITE)) {
727 /* 752 /*
728 * Without enhanced suppression-on-protection force 753 * Without enhanced suppression-on-protection force
729 * the dirty bit on for all writable ptes. 754 * the dirty bit on for all writable ptes.
730 */ 755 */
731 pte_val(entry) |= _PAGE_SWC; 756 pte_val(entry) |= _PAGE_DIRTY;
732 pte_val(entry) &= ~_PAGE_RO; 757 pte_val(entry) &= ~_PAGE_PROTECT;
733 } 758 }
734 *ptep = entry; 759 *ptep = entry;
735} 760}
@@ -841,21 +866,17 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
841 */ 866 */
842static inline int pte_write(pte_t pte) 867static inline int pte_write(pte_t pte)
843{ 868{
844 return (pte_val(pte) & _PAGE_SWW) != 0; 869 return (pte_val(pte) & _PAGE_WRITE) != 0;
845} 870}
846 871
847static inline int pte_dirty(pte_t pte) 872static inline int pte_dirty(pte_t pte)
848{ 873{
849 return (pte_val(pte) & _PAGE_SWC) != 0; 874 return (pte_val(pte) & _PAGE_DIRTY) != 0;
850} 875}
851 876
852static inline int pte_young(pte_t pte) 877static inline int pte_young(pte_t pte)
853{ 878{
854#ifdef CONFIG_PGSTE 879 return (pte_val(pte) & _PAGE_YOUNG) != 0;
855 if (pte_val(pte) & _PAGE_SWR)
856 return 1;
857#endif
858 return 0;
859} 880}
860 881
861/* 882/*
@@ -880,12 +901,12 @@ static inline void pud_clear(pud_t *pud)
880 901
881static inline void pmd_clear(pmd_t *pmdp) 902static inline void pmd_clear(pmd_t *pmdp)
882{ 903{
883 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY; 904 pmd_val(*pmdp) = _SEGMENT_ENTRY_INVALID;
884} 905}
885 906
886static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 907static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
887{ 908{
888 pte_val(*ptep) = _PAGE_TYPE_EMPTY; 909 pte_val(*ptep) = _PAGE_INVALID;
889} 910}
890 911
891/* 912/*
@@ -896,55 +917,63 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
896{ 917{
897 pte_val(pte) &= _PAGE_CHG_MASK; 918 pte_val(pte) &= _PAGE_CHG_MASK;
898 pte_val(pte) |= pgprot_val(newprot); 919 pte_val(pte) |= pgprot_val(newprot);
899 if ((pte_val(pte) & _PAGE_SWC) && (pte_val(pte) & _PAGE_SWW)) 920 /*
900 pte_val(pte) &= ~_PAGE_RO; 921 * newprot for PAGE_NONE, PAGE_READ and PAGE_WRITE has the
922 * invalid bit set, clear it again for readable, young pages
923 */
924 if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ))
925 pte_val(pte) &= ~_PAGE_INVALID;
926 /*
927 * newprot for PAGE_READ and PAGE_WRITE has the page protection
928 * bit set, clear it again for writable, dirty pages
929 */
930 if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE))
931 pte_val(pte) &= ~_PAGE_PROTECT;
901 return pte; 932 return pte;
902} 933}
903 934
904static inline pte_t pte_wrprotect(pte_t pte) 935static inline pte_t pte_wrprotect(pte_t pte)
905{ 936{
906 pte_val(pte) &= ~_PAGE_SWW; 937 pte_val(pte) &= ~_PAGE_WRITE;
907 /* Do not clobber _PAGE_TYPE_NONE pages! */ 938 pte_val(pte) |= _PAGE_PROTECT;
908 if (!(pte_val(pte) & _PAGE_INVALID))
909 pte_val(pte) |= _PAGE_RO;
910 return pte; 939 return pte;
911} 940}
912 941
913static inline pte_t pte_mkwrite(pte_t pte) 942static inline pte_t pte_mkwrite(pte_t pte)
914{ 943{
915 pte_val(pte) |= _PAGE_SWW; 944 pte_val(pte) |= _PAGE_WRITE;
916 if (pte_val(pte) & _PAGE_SWC) 945 if (pte_val(pte) & _PAGE_DIRTY)
917 pte_val(pte) &= ~_PAGE_RO; 946 pte_val(pte) &= ~_PAGE_PROTECT;
918 return pte; 947 return pte;
919} 948}
920 949
921static inline pte_t pte_mkclean(pte_t pte) 950static inline pte_t pte_mkclean(pte_t pte)
922{ 951{
923 pte_val(pte) &= ~_PAGE_SWC; 952 pte_val(pte) &= ~_PAGE_DIRTY;
924 /* Do not clobber _PAGE_TYPE_NONE pages! */ 953 pte_val(pte) |= _PAGE_PROTECT;
925 if (!(pte_val(pte) & _PAGE_INVALID))
926 pte_val(pte) |= _PAGE_RO;
927 return pte; 954 return pte;
928} 955}
929 956
930static inline pte_t pte_mkdirty(pte_t pte) 957static inline pte_t pte_mkdirty(pte_t pte)
931{ 958{
932 pte_val(pte) |= _PAGE_SWC; 959 pte_val(pte) |= _PAGE_DIRTY;
933 if (pte_val(pte) & _PAGE_SWW) 960 if (pte_val(pte) & _PAGE_WRITE)
934 pte_val(pte) &= ~_PAGE_RO; 961 pte_val(pte) &= ~_PAGE_PROTECT;
935 return pte; 962 return pte;
936} 963}
937 964
938static inline pte_t pte_mkold(pte_t pte) 965static inline pte_t pte_mkold(pte_t pte)
939{ 966{
940#ifdef CONFIG_PGSTE 967 pte_val(pte) &= ~_PAGE_YOUNG;
941 pte_val(pte) &= ~_PAGE_SWR; 968 pte_val(pte) |= _PAGE_INVALID;
942#endif
943 return pte; 969 return pte;
944} 970}
945 971
946static inline pte_t pte_mkyoung(pte_t pte) 972static inline pte_t pte_mkyoung(pte_t pte)
947{ 973{
974 pte_val(pte) |= _PAGE_YOUNG;
975 if (pte_val(pte) & _PAGE_READ)
976 pte_val(pte) &= ~_PAGE_INVALID;
948 return pte; 977 return pte;
949} 978}
950 979
@@ -957,7 +986,7 @@ static inline pte_t pte_mkspecial(pte_t pte)
957#ifdef CONFIG_HUGETLB_PAGE 986#ifdef CONFIG_HUGETLB_PAGE
958static inline pte_t pte_mkhuge(pte_t pte) 987static inline pte_t pte_mkhuge(pte_t pte)
959{ 988{
960 pte_val(pte) |= (_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO); 989 pte_val(pte) |= _PAGE_LARGE;
961 return pte; 990 return pte;
962} 991}
963#endif 992#endif
@@ -974,8 +1003,8 @@ static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm,
974 if (mm_has_pgste(mm)) { 1003 if (mm_has_pgste(mm)) {
975 pgste = pgste_get_lock(ptep); 1004 pgste = pgste_get_lock(ptep);
976 pgste = pgste_update_all(ptep, pgste); 1005 pgste = pgste_update_all(ptep, pgste);
977 dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); 1006 dirty = !!(pgste_val(pgste) & PGSTE_HC_BIT);
978 pgste_val(pgste) &= ~PGSTE_UC_BIT; 1007 pgste_val(pgste) &= ~PGSTE_HC_BIT;
979 pgste_set_unlock(ptep, pgste); 1008 pgste_set_unlock(ptep, pgste);
980 return dirty; 1009 return dirty;
981 } 1010 }
@@ -994,59 +1023,75 @@ static inline int ptep_test_and_clear_user_young(struct mm_struct *mm,
994 if (mm_has_pgste(mm)) { 1023 if (mm_has_pgste(mm)) {
995 pgste = pgste_get_lock(ptep); 1024 pgste = pgste_get_lock(ptep);
996 pgste = pgste_update_young(ptep, pgste); 1025 pgste = pgste_update_young(ptep, pgste);
997 young = !!(pgste_val(pgste) & PGSTE_UR_BIT); 1026 young = !!(pgste_val(pgste) & PGSTE_HR_BIT);
998 pgste_val(pgste) &= ~PGSTE_UR_BIT; 1027 pgste_val(pgste) &= ~PGSTE_HR_BIT;
999 pgste_set_unlock(ptep, pgste); 1028 pgste_set_unlock(ptep, pgste);
1000 } 1029 }
1001 return young; 1030 return young;
1002} 1031}
1003 1032
1033static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
1034{
1035 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1036#ifndef CONFIG_64BIT
1037 /* pto must point to the start of the segment table */
1038 pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
1039#else
1040 /* ipte in zarch mode can do the math */
1041 pte_t *pto = ptep;
1042#endif
1043 asm volatile(
1044 " ipte %2,%3"
1045 : "=m" (*ptep) : "m" (*ptep),
1046 "a" (pto), "a" (address));
1047 }
1048}
1049
1050static inline void ptep_flush_lazy(struct mm_struct *mm,
1051 unsigned long address, pte_t *ptep)
1052{
1053 int active = (mm == current->active_mm) ? 1 : 0;
1054
1055 if (atomic_read(&mm->context.attach_count) > active)
1056 __ptep_ipte(address, ptep);
1057 else
1058 mm->context.flush_mm = 1;
1059}
1060
1004#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 1061#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1005static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, 1062static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1006 unsigned long addr, pte_t *ptep) 1063 unsigned long addr, pte_t *ptep)
1007{ 1064{
1008 pgste_t pgste; 1065 pgste_t pgste;
1009 pte_t pte; 1066 pte_t pte;
1067 int young;
1010 1068
1011 if (mm_has_pgste(vma->vm_mm)) { 1069 if (mm_has_pgste(vma->vm_mm)) {
1012 pgste = pgste_get_lock(ptep); 1070 pgste = pgste_get_lock(ptep);
1013 pgste = pgste_update_young(ptep, pgste); 1071 pgste = pgste_ipte_notify(vma->vm_mm, addr, ptep, pgste);
1014 pte = *ptep;
1015 *ptep = pte_mkold(pte);
1016 pgste_set_unlock(ptep, pgste);
1017 return pte_young(pte);
1018 } 1072 }
1019 return 0; 1073
1074 pte = *ptep;
1075 __ptep_ipte(addr, ptep);
1076 young = pte_young(pte);
1077 pte = pte_mkold(pte);
1078
1079 if (mm_has_pgste(vma->vm_mm)) {
1080 pgste_set_pte(ptep, pte);
1081 pgste_set_unlock(ptep, pgste);
1082 } else
1083 *ptep = pte;
1084
1085 return young;
1020} 1086}
1021 1087
1022#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 1088#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1023static inline int ptep_clear_flush_young(struct vm_area_struct *vma, 1089static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1024 unsigned long address, pte_t *ptep) 1090 unsigned long address, pte_t *ptep)
1025{ 1091{
1026 /* No need to flush TLB
1027 * On s390 reference bits are in storage key and never in TLB
1028 * With virtualization we handle the reference bit, without we
1029 * we can simply return */
1030 return ptep_test_and_clear_young(vma, address, ptep); 1092 return ptep_test_and_clear_young(vma, address, ptep);
1031} 1093}
1032 1094
1033static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
1034{
1035 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
1036#ifndef CONFIG_64BIT
1037 /* pto must point to the start of the segment table */
1038 pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
1039#else
1040 /* ipte in zarch mode can do the math */
1041 pte_t *pto = ptep;
1042#endif
1043 asm volatile(
1044 " ipte %2,%3"
1045 : "=m" (*ptep) : "m" (*ptep),
1046 "a" (pto), "a" (address));
1047 }
1048}
1049
1050/* 1095/*
1051 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush 1096 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1052 * both clear the TLB for the unmapped pte. The reason is that 1097 * both clear the TLB for the unmapped pte. The reason is that
@@ -1067,16 +1112,14 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1067 pgste_t pgste; 1112 pgste_t pgste;
1068 pte_t pte; 1113 pte_t pte;
1069 1114
1070 mm->context.flush_mm = 1;
1071 if (mm_has_pgste(mm)) { 1115 if (mm_has_pgste(mm)) {
1072 pgste = pgste_get_lock(ptep); 1116 pgste = pgste_get_lock(ptep);
1073 pgste = pgste_ipte_notify(mm, address, ptep, pgste); 1117 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1074 } 1118 }
1075 1119
1076 pte = *ptep; 1120 pte = *ptep;
1077 if (!mm_exclusive(mm)) 1121 ptep_flush_lazy(mm, address, ptep);
1078 __ptep_ipte(address, ptep); 1122 pte_val(*ptep) = _PAGE_INVALID;
1079 pte_val(*ptep) = _PAGE_TYPE_EMPTY;
1080 1123
1081 if (mm_has_pgste(mm)) { 1124 if (mm_has_pgste(mm)) {
1082 pgste = pgste_update_all(&pte, pgste); 1125 pgste = pgste_update_all(&pte, pgste);
@@ -1093,15 +1136,14 @@ static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
1093 pgste_t pgste; 1136 pgste_t pgste;
1094 pte_t pte; 1137 pte_t pte;
1095 1138
1096 mm->context.flush_mm = 1;
1097 if (mm_has_pgste(mm)) { 1139 if (mm_has_pgste(mm)) {
1098 pgste = pgste_get_lock(ptep); 1140 pgste = pgste_get_lock(ptep);
1099 pgste_ipte_notify(mm, address, ptep, pgste); 1141 pgste_ipte_notify(mm, address, ptep, pgste);
1100 } 1142 }
1101 1143
1102 pte = *ptep; 1144 pte = *ptep;
1103 if (!mm_exclusive(mm)) 1145 ptep_flush_lazy(mm, address, ptep);
1104 __ptep_ipte(address, ptep); 1146 pte_val(*ptep) |= _PAGE_INVALID;
1105 1147
1106 if (mm_has_pgste(mm)) { 1148 if (mm_has_pgste(mm)) {
1107 pgste = pgste_update_all(&pte, pgste); 1149 pgste = pgste_update_all(&pte, pgste);
@@ -1117,7 +1159,7 @@ static inline void ptep_modify_prot_commit(struct mm_struct *mm,
1117 pgste_t pgste; 1159 pgste_t pgste;
1118 1160
1119 if (mm_has_pgste(mm)) { 1161 if (mm_has_pgste(mm)) {
1120 pgste = *(pgste_t *)(ptep + PTRS_PER_PTE); 1162 pgste = pgste_get(ptep);
1121 pgste_set_key(ptep, pgste, pte); 1163 pgste_set_key(ptep, pgste, pte);
1122 pgste_set_pte(ptep, pte); 1164 pgste_set_pte(ptep, pte);
1123 pgste_set_unlock(ptep, pgste); 1165 pgste_set_unlock(ptep, pgste);
@@ -1139,7 +1181,7 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1139 1181
1140 pte = *ptep; 1182 pte = *ptep;
1141 __ptep_ipte(address, ptep); 1183 __ptep_ipte(address, ptep);
1142 pte_val(*ptep) = _PAGE_TYPE_EMPTY; 1184 pte_val(*ptep) = _PAGE_INVALID;
1143 1185
1144 if (mm_has_pgste(vma->vm_mm)) { 1186 if (mm_has_pgste(vma->vm_mm)) {
1145 pgste = pgste_update_all(&pte, pgste); 1187 pgste = pgste_update_all(&pte, pgste);
@@ -1163,18 +1205,17 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1163 pgste_t pgste; 1205 pgste_t pgste;
1164 pte_t pte; 1206 pte_t pte;
1165 1207
1166 if (mm_has_pgste(mm)) { 1208 if (!full && mm_has_pgste(mm)) {
1167 pgste = pgste_get_lock(ptep); 1209 pgste = pgste_get_lock(ptep);
1168 if (!full) 1210 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1169 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1170 } 1211 }
1171 1212
1172 pte = *ptep; 1213 pte = *ptep;
1173 if (!full) 1214 if (!full)
1174 __ptep_ipte(address, ptep); 1215 ptep_flush_lazy(mm, address, ptep);
1175 pte_val(*ptep) = _PAGE_TYPE_EMPTY; 1216 pte_val(*ptep) = _PAGE_INVALID;
1176 1217
1177 if (mm_has_pgste(mm)) { 1218 if (!full && mm_has_pgste(mm)) {
1178 pgste = pgste_update_all(&pte, pgste); 1219 pgste = pgste_update_all(&pte, pgste);
1179 pgste_set_unlock(ptep, pgste); 1220 pgste_set_unlock(ptep, pgste);
1180 } 1221 }
@@ -1189,14 +1230,12 @@ static inline pte_t ptep_set_wrprotect(struct mm_struct *mm,
1189 pte_t pte = *ptep; 1230 pte_t pte = *ptep;
1190 1231
1191 if (pte_write(pte)) { 1232 if (pte_write(pte)) {
1192 mm->context.flush_mm = 1;
1193 if (mm_has_pgste(mm)) { 1233 if (mm_has_pgste(mm)) {
1194 pgste = pgste_get_lock(ptep); 1234 pgste = pgste_get_lock(ptep);
1195 pgste = pgste_ipte_notify(mm, address, ptep, pgste); 1235 pgste = pgste_ipte_notify(mm, address, ptep, pgste);
1196 } 1236 }
1197 1237
1198 if (!mm_exclusive(mm)) 1238 ptep_flush_lazy(mm, address, ptep);
1199 __ptep_ipte(address, ptep);
1200 pte = pte_wrprotect(pte); 1239 pte = pte_wrprotect(pte);
1201 1240
1202 if (mm_has_pgste(mm)) { 1241 if (mm_has_pgste(mm)) {
@@ -1240,7 +1279,7 @@ static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1240{ 1279{
1241 pte_t __pte; 1280 pte_t __pte;
1242 pte_val(__pte) = physpage + pgprot_val(pgprot); 1281 pte_val(__pte) = physpage + pgprot_val(pgprot);
1243 return __pte; 1282 return pte_mkyoung(__pte);
1244} 1283}
1245 1284
1246static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) 1285static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
@@ -1248,10 +1287,8 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1248 unsigned long physpage = page_to_phys(page); 1287 unsigned long physpage = page_to_phys(page);
1249 pte_t __pte = mk_pte_phys(physpage, pgprot); 1288 pte_t __pte = mk_pte_phys(physpage, pgprot);
1250 1289
1251 if ((pte_val(__pte) & _PAGE_SWW) && PageDirty(page)) { 1290 if (pte_write(__pte) && PageDirty(page))
1252 pte_val(__pte) |= _PAGE_SWC; 1291 __pte = pte_mkdirty(__pte);
1253 pte_val(__pte) &= ~_PAGE_RO;
1254 }
1255 return __pte; 1292 return __pte;
1256} 1293}
1257 1294
@@ -1313,7 +1350,7 @@ static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
1313 unsigned long sto = (unsigned long) pmdp - 1350 unsigned long sto = (unsigned long) pmdp -
1314 pmd_index(address) * sizeof(pmd_t); 1351 pmd_index(address) * sizeof(pmd_t);
1315 1352
1316 if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) { 1353 if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)) {
1317 asm volatile( 1354 asm volatile(
1318 " .insn rrf,0xb98e0000,%2,%3,0,0" 1355 " .insn rrf,0xb98e0000,%2,%3,0,0"
1319 : "=m" (*pmdp) 1356 : "=m" (*pmdp)
@@ -1324,24 +1361,68 @@ static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
1324 } 1361 }
1325} 1362}
1326 1363
1364static inline void __pmd_csp(pmd_t *pmdp)
1365{
1366 register unsigned long reg2 asm("2") = pmd_val(*pmdp);
1367 register unsigned long reg3 asm("3") = pmd_val(*pmdp) |
1368 _SEGMENT_ENTRY_INVALID;
1369 register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5;
1370
1371 asm volatile(
1372 " csp %1,%3"
1373 : "=m" (*pmdp)
1374 : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc");
1375}
1376
1327#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) 1377#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
1328static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot) 1378static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1329{ 1379{
1330 /* 1380 /*
1331 * pgprot is PAGE_NONE, PAGE_RO, or PAGE_RW (see __Pxxx / __Sxxx) 1381 * pgprot is PAGE_NONE, PAGE_READ, or PAGE_WRITE (see __Pxxx / __Sxxx)
1332 * Convert to segment table entry format. 1382 * Convert to segment table entry format.
1333 */ 1383 */
1334 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE)) 1384 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1335 return pgprot_val(SEGMENT_NONE); 1385 return pgprot_val(SEGMENT_NONE);
1336 if (pgprot_val(pgprot) == pgprot_val(PAGE_RO)) 1386 if (pgprot_val(pgprot) == pgprot_val(PAGE_READ))
1337 return pgprot_val(SEGMENT_RO); 1387 return pgprot_val(SEGMENT_READ);
1338 return pgprot_val(SEGMENT_RW); 1388 return pgprot_val(SEGMENT_WRITE);
1389}
1390
1391static inline pmd_t pmd_mkyoung(pmd_t pmd)
1392{
1393#ifdef CONFIG_64BIT
1394 if (pmd_prot_none(pmd)) {
1395 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
1396 } else {
1397 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG;
1398 pmd_val(pmd) &= ~_SEGMENT_ENTRY_INVALID;
1399 }
1400#endif
1401 return pmd;
1402}
1403
1404static inline pmd_t pmd_mkold(pmd_t pmd)
1405{
1406#ifdef CONFIG_64BIT
1407 if (pmd_prot_none(pmd)) {
1408 pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT;
1409 } else {
1410 pmd_val(pmd) &= ~_SEGMENT_ENTRY_YOUNG;
1411 pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID;
1412 }
1413#endif
1414 return pmd;
1339} 1415}
1340 1416
1341static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) 1417static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1342{ 1418{
1419 int young;
1420
1421 young = pmd_young(pmd);
1343 pmd_val(pmd) &= _SEGMENT_CHG_MASK; 1422 pmd_val(pmd) &= _SEGMENT_CHG_MASK;
1344 pmd_val(pmd) |= massage_pgprot_pmd(newprot); 1423 pmd_val(pmd) |= massage_pgprot_pmd(newprot);
1424 if (young)
1425 pmd = pmd_mkyoung(pmd);
1345 return pmd; 1426 return pmd;
1346} 1427}
1347 1428
@@ -1349,14 +1430,14 @@ static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1349{ 1430{
1350 pmd_t __pmd; 1431 pmd_t __pmd;
1351 pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot); 1432 pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
1352 return __pmd; 1433 return pmd_mkyoung(__pmd);
1353} 1434}
1354 1435
1355static inline pmd_t pmd_mkwrite(pmd_t pmd) 1436static inline pmd_t pmd_mkwrite(pmd_t pmd)
1356{ 1437{
1357 /* Do not clobber _HPAGE_TYPE_NONE pages! */ 1438 /* Do not clobber PROT_NONE segments! */
1358 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_INV)) 1439 if (!pmd_prot_none(pmd))
1359 pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO; 1440 pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT;
1360 return pmd; 1441 return pmd;
1361} 1442}
1362#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */ 1443#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */
@@ -1378,7 +1459,7 @@ static inline int pmd_trans_splitting(pmd_t pmd)
1378static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, 1459static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1379 pmd_t *pmdp, pmd_t entry) 1460 pmd_t *pmdp, pmd_t entry)
1380{ 1461{
1381 if (!(pmd_val(entry) & _SEGMENT_ENTRY_INV) && MACHINE_HAS_EDAT1) 1462 if (!(pmd_val(entry) & _SEGMENT_ENTRY_INVALID) && MACHINE_HAS_EDAT1)
1382 pmd_val(entry) |= _SEGMENT_ENTRY_CO; 1463 pmd_val(entry) |= _SEGMENT_ENTRY_CO;
1383 *pmdp = entry; 1464 *pmdp = entry;
1384} 1465}
@@ -1391,7 +1472,9 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd)
1391 1472
1392static inline pmd_t pmd_wrprotect(pmd_t pmd) 1473static inline pmd_t pmd_wrprotect(pmd_t pmd)
1393{ 1474{
1394 pmd_val(pmd) |= _SEGMENT_ENTRY_RO; 1475 /* Do not clobber PROT_NONE segments! */
1476 if (!pmd_prot_none(pmd))
1477 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
1395 return pmd; 1478 return pmd;
1396} 1479}
1397 1480
@@ -1401,50 +1484,16 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd)
1401 return pmd; 1484 return pmd;
1402} 1485}
1403 1486
1404static inline pmd_t pmd_mkold(pmd_t pmd)
1405{
1406 /* No referenced bit in the segment table entry. */
1407 return pmd;
1408}
1409
1410static inline pmd_t pmd_mkyoung(pmd_t pmd)
1411{
1412 /* No referenced bit in the segment table entry. */
1413 return pmd;
1414}
1415
1416#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 1487#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1417static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 1488static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1418 unsigned long address, pmd_t *pmdp) 1489 unsigned long address, pmd_t *pmdp)
1419{ 1490{
1420 unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK; 1491 pmd_t pmd;
1421 long tmp, rc;
1422 int counter;
1423 1492
1424 rc = 0; 1493 pmd = *pmdp;
1425 if (MACHINE_HAS_RRBM) { 1494 __pmd_idte(address, pmdp);
1426 counter = PTRS_PER_PTE >> 6; 1495 *pmdp = pmd_mkold(pmd);
1427 asm volatile( 1496 return pmd_young(pmd);
1428 "0: .insn rre,0xb9ae0000,%0,%3\n" /* rrbm */
1429 " ogr %1,%0\n"
1430 " la %3,0(%4,%3)\n"
1431 " brct %2,0b\n"
1432 : "=&d" (tmp), "+&d" (rc), "+d" (counter),
1433 "+a" (pmd_addr)
1434 : "a" (64 * 4096UL) : "cc");
1435 rc = !!rc;
1436 } else {
1437 counter = PTRS_PER_PTE;
1438 asm volatile(
1439 "0: rrbe 0,%2\n"
1440 " la %2,0(%3,%2)\n"
1441 " brc 12,1f\n"
1442 " lhi %0,1\n"
1443 "1: brct %1,0b\n"
1444 : "+d" (rc), "+d" (counter), "+a" (pmd_addr)
1445 : "a" (4096UL) : "cc");
1446 }
1447 return rc;
1448} 1497}
1449 1498
1450#define __HAVE_ARCH_PMDP_GET_AND_CLEAR 1499#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
@@ -1510,10 +1559,8 @@ static inline unsigned long pmd_pfn(pmd_t pmd)
1510 * exception will occur instead of a page translation exception. The 1559 * exception will occur instead of a page translation exception. The
1511 * specifiation exception has the bad habit not to store necessary 1560 * specifiation exception has the bad habit not to store necessary
1512 * information in the lowcore. 1561 * information in the lowcore.
1513 * Bit 21 and bit 22 are the page invalid bit and the page protection 1562 * Bits 21, 22, 30 and 31 are used to indicate the page type.
1514 * bit. We set both to indicate a swapped page. 1563 * A swap pte is indicated by bit pattern (pte & 0x603) == 0x402
1515 * Bit 30 and 31 are used to distinguish the different page types. For
1516 * a swapped page these bits need to be zero.
1517 * This leaves the bits 1-19 and bits 24-29 to store type and offset. 1564 * This leaves the bits 1-19 and bits 24-29 to store type and offset.
1518 * We use the 5 bits from 25-29 for the type and the 20 bits from 1-19 1565 * We use the 5 bits from 25-29 for the type and the 20 bits from 1-19
1519 * plus 24 for the offset. 1566 * plus 24 for the offset.
@@ -1527,10 +1574,8 @@ static inline unsigned long pmd_pfn(pmd_t pmd)
1527 * exception will occur instead of a page translation exception. The 1574 * exception will occur instead of a page translation exception. The
1528 * specifiation exception has the bad habit not to store necessary 1575 * specifiation exception has the bad habit not to store necessary
1529 * information in the lowcore. 1576 * information in the lowcore.
1530 * Bit 53 and bit 54 are the page invalid bit and the page protection 1577 * Bits 53, 54, 62 and 63 are used to indicate the page type.
1531 * bit. We set both to indicate a swapped page. 1578 * A swap pte is indicated by bit pattern (pte & 0x603) == 0x402
1532 * Bit 62 and 63 are used to distinguish the different page types. For
1533 * a swapped page these bits need to be zero.
1534 * This leaves the bits 0-51 and bits 56-61 to store type and offset. 1579 * This leaves the bits 0-51 and bits 56-61 to store type and offset.
1535 * We use the 5 bits from 57-61 for the type and the 53 bits from 0-51 1580 * We use the 5 bits from 57-61 for the type and the 53 bits from 0-51
1536 * plus 56 for the offset. 1581 * plus 56 for the offset.
@@ -1547,7 +1592,7 @@ static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1547{ 1592{
1548 pte_t pte; 1593 pte_t pte;
1549 offset &= __SWP_OFFSET_MASK; 1594 offset &= __SWP_OFFSET_MASK;
1550 pte_val(pte) = _PAGE_TYPE_SWAP | ((type & 0x1f) << 2) | 1595 pte_val(pte) = _PAGE_INVALID | _PAGE_TYPE | ((type & 0x1f) << 2) |
1551 ((offset & 1UL) << 7) | ((offset & ~1UL) << 11); 1596 ((offset & 1UL) << 7) | ((offset & ~1UL) << 11);
1552 return pte; 1597 return pte;
1553} 1598}
@@ -1570,7 +1615,7 @@ static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1570 1615
1571#define pgoff_to_pte(__off) \ 1616#define pgoff_to_pte(__off) \
1572 ((pte_t) { ((((__off) & 0x7f) << 1) + (((__off) >> 7) << 12)) \ 1617 ((pte_t) { ((((__off) & 0x7f) << 1) + (((__off) >> 7) << 12)) \
1573 | _PAGE_TYPE_FILE }) 1618 | _PAGE_INVALID | _PAGE_PROTECT })
1574 1619
1575#endif /* !__ASSEMBLY__ */ 1620#endif /* !__ASSEMBLY__ */
1576 1621
diff --git a/arch/s390/include/asm/serial.h b/arch/s390/include/asm/serial.h
new file mode 100644
index 000000000000..5b3e48ef534b
--- /dev/null
+++ b/arch/s390/include/asm/serial.h
@@ -0,0 +1,6 @@
1#ifndef _ASM_S390_SERIAL_H
2#define _ASM_S390_SERIAL_H
3
4#define BASE_BAUD 0
5
6#endif /* _ASM_S390_SERIAL_H */
diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h
index 80b6f11263c4..6dbd559763c9 100644
--- a/arch/s390/include/asm/switch_to.h
+++ b/arch/s390/include/asm/switch_to.h
@@ -8,6 +8,7 @@
8#define __ASM_SWITCH_TO_H 8#define __ASM_SWITCH_TO_H
9 9
10#include <linux/thread_info.h> 10#include <linux/thread_info.h>
11#include <asm/ptrace.h>
11 12
12extern struct task_struct *__switch_to(void *, void *); 13extern struct task_struct *__switch_to(void *, void *);
13extern void update_cr_regs(struct task_struct *task); 14extern void update_cr_regs(struct task_struct *task);
@@ -68,12 +69,16 @@ static inline void restore_fp_regs(s390_fp_regs *fpregs)
68 69
69static inline void save_access_regs(unsigned int *acrs) 70static inline void save_access_regs(unsigned int *acrs)
70{ 71{
71 asm volatile("stam 0,15,%0" : "=Q" (*acrs)); 72 typedef struct { int _[NUM_ACRS]; } acrstype;
73
74 asm volatile("stam 0,15,%0" : "=Q" (*(acrstype *)acrs));
72} 75}
73 76
74static inline void restore_access_regs(unsigned int *acrs) 77static inline void restore_access_regs(unsigned int *acrs)
75{ 78{
76 asm volatile("lam 0,15,%0" : : "Q" (*acrs)); 79 typedef struct { int _[NUM_ACRS]; } acrstype;
80
81 asm volatile("lam 0,15,%0" : : "Q" (*(acrstype *)acrs));
77} 82}
78 83
79#define switch_to(prev,next,last) do { \ 84#define switch_to(prev,next,last) do { \
diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 6d6d92b4ea11..2cb846c4b37f 100644
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -63,13 +63,14 @@ static inline void tlb_gather_mmu(struct mmu_gather *tlb,
63 63
64static inline void tlb_flush_mmu(struct mmu_gather *tlb) 64static inline void tlb_flush_mmu(struct mmu_gather *tlb)
65{ 65{
66 __tlb_flush_mm_lazy(tlb->mm);
66 tlb_table_flush(tlb); 67 tlb_table_flush(tlb);
67} 68}
68 69
69static inline void tlb_finish_mmu(struct mmu_gather *tlb, 70static inline void tlb_finish_mmu(struct mmu_gather *tlb,
70 unsigned long start, unsigned long end) 71 unsigned long start, unsigned long end)
71{ 72{
72 tlb_table_flush(tlb); 73 tlb_flush_mmu(tlb);
73} 74}
74 75
75/* 76/*
diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h
index 6b32af30878c..f9fef0425fee 100644
--- a/arch/s390/include/asm/tlbflush.h
+++ b/arch/s390/include/asm/tlbflush.h
@@ -86,7 +86,7 @@ static inline void __tlb_flush_mm(struct mm_struct * mm)
86 __tlb_flush_full(mm); 86 __tlb_flush_full(mm);
87} 87}
88 88
89static inline void __tlb_flush_mm_cond(struct mm_struct * mm) 89static inline void __tlb_flush_mm_lazy(struct mm_struct * mm)
90{ 90{
91 if (mm->context.flush_mm) { 91 if (mm->context.flush_mm) {
92 __tlb_flush_mm(mm); 92 __tlb_flush_mm(mm);
@@ -118,13 +118,13 @@ static inline void __tlb_flush_mm_cond(struct mm_struct * mm)
118 118
119static inline void flush_tlb_mm(struct mm_struct *mm) 119static inline void flush_tlb_mm(struct mm_struct *mm)
120{ 120{
121 __tlb_flush_mm_cond(mm); 121 __tlb_flush_mm_lazy(mm);
122} 122}
123 123
124static inline void flush_tlb_range(struct vm_area_struct *vma, 124static inline void flush_tlb_range(struct vm_area_struct *vma,
125 unsigned long start, unsigned long end) 125 unsigned long start, unsigned long end)
126{ 126{
127 __tlb_flush_mm_cond(vma->vm_mm); 127 __tlb_flush_mm_lazy(vma->vm_mm);
128} 128}
129 129
130static inline void flush_tlb_kernel_range(unsigned long start, 130static inline void flush_tlb_kernel_range(unsigned long start,
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index be7a408be7a1..cc30d1fb000c 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -18,6 +18,7 @@
18#include <asm/unistd.h> 18#include <asm/unistd.h>
19#include <asm/page.h> 19#include <asm/page.h>
20#include <asm/sigp.h> 20#include <asm/sigp.h>
21#include <asm/irq.h>
21 22
22__PT_R0 = __PT_GPRS 23__PT_R0 = __PT_GPRS
23__PT_R1 = __PT_GPRS + 4 24__PT_R1 = __PT_GPRS + 4
@@ -435,6 +436,11 @@ io_skip:
435io_loop: 436io_loop:
436 l %r1,BASED(.Ldo_IRQ) 437 l %r1,BASED(.Ldo_IRQ)
437 lr %r2,%r11 # pass pointer to pt_regs 438 lr %r2,%r11 # pass pointer to pt_regs
439 lhi %r3,IO_INTERRUPT
440 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
441 jz io_call
442 lhi %r3,THIN_INTERRUPT
443io_call:
438 basr %r14,%r1 # call do_IRQ 444 basr %r14,%r1 # call do_IRQ
439 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR 445 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR
440 jz io_return 446 jz io_return
@@ -584,9 +590,10 @@ ext_skip:
584 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR 590 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
585 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS 591 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
586 TRACE_IRQS_OFF 592 TRACE_IRQS_OFF
593 l %r1,BASED(.Ldo_IRQ)
587 lr %r2,%r11 # pass pointer to pt_regs 594 lr %r2,%r11 # pass pointer to pt_regs
588 l %r1,BASED(.Ldo_extint) 595 lhi %r3,EXT_INTERRUPT
589 basr %r14,%r1 # call do_extint 596 basr %r14,%r1 # call do_IRQ
590 j io_return 597 j io_return
591 598
592/* 599/*
@@ -879,13 +886,13 @@ cleanup_idle:
879 stm %r9,%r10,__LC_SYSTEM_TIMER 886 stm %r9,%r10,__LC_SYSTEM_TIMER
880 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) 887 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
881 # prepare return psw 888 # prepare return psw
882 n %r8,BASED(cleanup_idle_wait) # clear wait state bit 889 n %r8,BASED(cleanup_idle_wait) # clear irq & wait state bits
883 l %r9,24(%r11) # return from psw_idle 890 l %r9,24(%r11) # return from psw_idle
884 br %r14 891 br %r14
885cleanup_idle_insn: 892cleanup_idle_insn:
886 .long psw_idle_lpsw + 0x80000000 893 .long psw_idle_lpsw + 0x80000000
887cleanup_idle_wait: 894cleanup_idle_wait:
888 .long 0xfffdffff 895 .long 0xfcfdffff
889 896
890/* 897/*
891 * Integer constants 898 * Integer constants
@@ -902,7 +909,6 @@ cleanup_idle_wait:
902.Ldo_machine_check: .long s390_do_machine_check 909.Ldo_machine_check: .long s390_do_machine_check
903.Lhandle_mcck: .long s390_handle_mcck 910.Lhandle_mcck: .long s390_handle_mcck
904.Ldo_IRQ: .long do_IRQ 911.Ldo_IRQ: .long do_IRQ
905.Ldo_extint: .long do_extint
906.Ldo_signal: .long do_signal 912.Ldo_signal: .long do_signal
907.Ldo_notify_resume: .long do_notify_resume 913.Ldo_notify_resume: .long do_notify_resume
908.Ldo_per_trap: .long do_per_trap 914.Ldo_per_trap: .long do_per_trap
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 1c039d0c24c7..2b2188b97c6a 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -19,6 +19,7 @@
19#include <asm/unistd.h> 19#include <asm/unistd.h>
20#include <asm/page.h> 20#include <asm/page.h>
21#include <asm/sigp.h> 21#include <asm/sigp.h>
22#include <asm/irq.h>
22 23
23__PT_R0 = __PT_GPRS 24__PT_R0 = __PT_GPRS
24__PT_R1 = __PT_GPRS + 8 25__PT_R1 = __PT_GPRS + 8
@@ -468,6 +469,11 @@ io_skip:
468 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 469 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
469io_loop: 470io_loop:
470 lgr %r2,%r11 # pass pointer to pt_regs 471 lgr %r2,%r11 # pass pointer to pt_regs
472 lghi %r3,IO_INTERRUPT
473 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
474 jz io_call
475 lghi %r3,THIN_INTERRUPT
476io_call:
471 brasl %r14,do_IRQ 477 brasl %r14,do_IRQ
472 tm __LC_MACHINE_FLAGS+6,0x10 # MACHINE_FLAG_LPAR 478 tm __LC_MACHINE_FLAGS+6,0x10 # MACHINE_FLAG_LPAR
473 jz io_return 479 jz io_return
@@ -623,7 +629,8 @@ ext_skip:
623 TRACE_IRQS_OFF 629 TRACE_IRQS_OFF
624 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) 630 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
625 lgr %r2,%r11 # pass pointer to pt_regs 631 lgr %r2,%r11 # pass pointer to pt_regs
626 brasl %r14,do_extint 632 lghi %r3,EXT_INTERRUPT
633 brasl %r14,do_IRQ
627 j io_return 634 j io_return
628 635
629/* 636/*
@@ -922,7 +929,7 @@ cleanup_idle:
922 stg %r9,__LC_SYSTEM_TIMER 929 stg %r9,__LC_SYSTEM_TIMER
923 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) 930 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
924 # prepare return psw 931 # prepare return psw
925 nihh %r8,0xfffd # clear wait state bit 932 nihh %r8,0xfcfd # clear irq & wait state bits
926 lg %r9,48(%r11) # return from psw_idle 933 lg %r9,48(%r11) # return from psw_idle
927 br %r14 934 br %r14
928cleanup_idle_insn: 935cleanup_idle_insn:
diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
index 54b0995514e8..b34ba0ea96a9 100644
--- a/arch/s390/kernel/irq.c
+++ b/arch/s390/kernel/irq.c
@@ -22,6 +22,7 @@
22#include <asm/cputime.h> 22#include <asm/cputime.h>
23#include <asm/lowcore.h> 23#include <asm/lowcore.h>
24#include <asm/irq.h> 24#include <asm/irq.h>
25#include <asm/hw_irq.h>
25#include "entry.h" 26#include "entry.h"
26 27
27DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); 28DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat);
@@ -42,9 +43,10 @@ struct irq_class {
42 * Since the external and I/O interrupt fields are already sums we would end 43 * Since the external and I/O interrupt fields are already sums we would end
43 * up with having a sum which accounts each interrupt twice. 44 * up with having a sum which accounts each interrupt twice.
44 */ 45 */
45static const struct irq_class irqclass_main_desc[NR_IRQS] = { 46static const struct irq_class irqclass_main_desc[NR_IRQS_BASE] = {
46 [EXTERNAL_INTERRUPT] = {.name = "EXT"}, 47 [EXT_INTERRUPT] = {.name = "EXT"},
47 [IO_INTERRUPT] = {.name = "I/O"} 48 [IO_INTERRUPT] = {.name = "I/O"},
49 [THIN_INTERRUPT] = {.name = "AIO"},
48}; 50};
49 51
50/* 52/*
@@ -86,6 +88,28 @@ static const struct irq_class irqclass_sub_desc[NR_ARCH_IRQS] = {
86 [CPU_RST] = {.name = "RST", .desc = "[CPU] CPU Restart"}, 88 [CPU_RST] = {.name = "RST", .desc = "[CPU] CPU Restart"},
87}; 89};
88 90
91void __init init_IRQ(void)
92{
93 irq_reserve_irqs(0, THIN_INTERRUPT);
94 init_cio_interrupts();
95 init_airq_interrupts();
96 init_ext_interrupts();
97}
98
99void do_IRQ(struct pt_regs *regs, int irq)
100{
101 struct pt_regs *old_regs;
102
103 old_regs = set_irq_regs(regs);
104 irq_enter();
105 if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator)
106 /* Serve timer interrupts first. */
107 clock_comparator_work();
108 generic_handle_irq(irq);
109 irq_exit();
110 set_irq_regs(old_regs);
111}
112
89/* 113/*
90 * show_interrupts is needed by /proc/interrupts. 114 * show_interrupts is needed by /proc/interrupts.
91 */ 115 */
@@ -100,27 +124,36 @@ int show_interrupts(struct seq_file *p, void *v)
100 for_each_online_cpu(cpu) 124 for_each_online_cpu(cpu)
101 seq_printf(p, "CPU%d ", cpu); 125 seq_printf(p, "CPU%d ", cpu);
102 seq_putc(p, '\n'); 126 seq_putc(p, '\n');
127 goto out;
103 } 128 }
104 if (irq < NR_IRQS) { 129 if (irq < NR_IRQS) {
130 if (irq >= NR_IRQS_BASE)
131 goto out;
105 seq_printf(p, "%s: ", irqclass_main_desc[irq].name); 132 seq_printf(p, "%s: ", irqclass_main_desc[irq].name);
106 for_each_online_cpu(cpu) 133 for_each_online_cpu(cpu)
107 seq_printf(p, "%10u ", kstat_cpu(cpu).irqs[irq]); 134 seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu));
108 seq_putc(p, '\n'); 135 seq_putc(p, '\n');
109 goto skip_arch_irqs; 136 goto out;
110 } 137 }
111 for (irq = 0; irq < NR_ARCH_IRQS; irq++) { 138 for (irq = 0; irq < NR_ARCH_IRQS; irq++) {
112 seq_printf(p, "%s: ", irqclass_sub_desc[irq].name); 139 seq_printf(p, "%s: ", irqclass_sub_desc[irq].name);
113 for_each_online_cpu(cpu) 140 for_each_online_cpu(cpu)
114 seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).irqs[irq]); 141 seq_printf(p, "%10u ",
142 per_cpu(irq_stat, cpu).irqs[irq]);
115 if (irqclass_sub_desc[irq].desc) 143 if (irqclass_sub_desc[irq].desc)
116 seq_printf(p, " %s", irqclass_sub_desc[irq].desc); 144 seq_printf(p, " %s", irqclass_sub_desc[irq].desc);
117 seq_putc(p, '\n'); 145 seq_putc(p, '\n');
118 } 146 }
119skip_arch_irqs: 147out:
120 put_online_cpus(); 148 put_online_cpus();
121 return 0; 149 return 0;
122} 150}
123 151
152int arch_show_interrupts(struct seq_file *p, int prec)
153{
154 return 0;
155}
156
124/* 157/*
125 * Switch to the asynchronous interrupt stack for softirq execution. 158 * Switch to the asynchronous interrupt stack for softirq execution.
126 */ 159 */
@@ -159,14 +192,6 @@ asmlinkage void do_softirq(void)
159 local_irq_restore(flags); 192 local_irq_restore(flags);
160} 193}
161 194
162#ifdef CONFIG_PROC_FS
163void init_irq_proc(void)
164{
165 if (proc_mkdir("irq", NULL))
166 create_prof_cpu_mask();
167}
168#endif
169
170/* 195/*
171 * ext_int_hash[index] is the list head for all external interrupts that hash 196 * ext_int_hash[index] is the list head for all external interrupts that hash
172 * to this index. 197 * to this index.
@@ -183,14 +208,6 @@ struct ext_int_info {
183/* ext_int_hash_lock protects the handler lists for external interrupts */ 208/* ext_int_hash_lock protects the handler lists for external interrupts */
184DEFINE_SPINLOCK(ext_int_hash_lock); 209DEFINE_SPINLOCK(ext_int_hash_lock);
185 210
186static void __init init_external_interrupts(void)
187{
188 int idx;
189
190 for (idx = 0; idx < ARRAY_SIZE(ext_int_hash); idx++)
191 INIT_LIST_HEAD(&ext_int_hash[idx]);
192}
193
194static inline int ext_hash(u16 code) 211static inline int ext_hash(u16 code)
195{ 212{
196 return (code + (code >> 9)) & 0xff; 213 return (code + (code >> 9)) & 0xff;
@@ -234,20 +251,13 @@ int unregister_external_interrupt(u16 code, ext_int_handler_t handler)
234} 251}
235EXPORT_SYMBOL(unregister_external_interrupt); 252EXPORT_SYMBOL(unregister_external_interrupt);
236 253
237void __irq_entry do_extint(struct pt_regs *regs) 254static irqreturn_t do_ext_interrupt(int irq, void *dummy)
238{ 255{
256 struct pt_regs *regs = get_irq_regs();
239 struct ext_code ext_code; 257 struct ext_code ext_code;
240 struct pt_regs *old_regs;
241 struct ext_int_info *p; 258 struct ext_int_info *p;
242 int index; 259 int index;
243 260
244 old_regs = set_irq_regs(regs);
245 irq_enter();
246 if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) {
247 /* Serve timer interrupts first. */
248 clock_comparator_work();
249 }
250 kstat_incr_irqs_this_cpu(EXTERNAL_INTERRUPT, NULL);
251 ext_code = *(struct ext_code *) &regs->int_code; 261 ext_code = *(struct ext_code *) &regs->int_code;
252 if (ext_code.code != 0x1004) 262 if (ext_code.code != 0x1004)
253 __get_cpu_var(s390_idle).nohz_delay = 1; 263 __get_cpu_var(s390_idle).nohz_delay = 1;
@@ -259,13 +269,25 @@ void __irq_entry do_extint(struct pt_regs *regs)
259 p->handler(ext_code, regs->int_parm, 269 p->handler(ext_code, regs->int_parm,
260 regs->int_parm_long); 270 regs->int_parm_long);
261 rcu_read_unlock(); 271 rcu_read_unlock();
262 irq_exit(); 272
263 set_irq_regs(old_regs); 273 return IRQ_HANDLED;
264} 274}
265 275
266void __init init_IRQ(void) 276static struct irqaction external_interrupt = {
277 .name = "EXT",
278 .handler = do_ext_interrupt,
279};
280
281void __init init_ext_interrupts(void)
267{ 282{
268 init_external_interrupts(); 283 int idx;
284
285 for (idx = 0; idx < ARRAY_SIZE(ext_int_hash); idx++)
286 INIT_LIST_HEAD(&ext_int_hash[idx]);
287
288 irq_set_chip_and_handler(EXT_INTERRUPT,
289 &dummy_irq_chip, handle_percpu_irq);
290 setup_irq(EXT_INTERRUPT, &external_interrupt);
269} 291}
270 292
271static DEFINE_SPINLOCK(sc_irq_lock); 293static DEFINE_SPINLOCK(sc_irq_lock);
@@ -313,69 +335,3 @@ void measurement_alert_subclass_unregister(void)
313 spin_unlock(&ma_subclass_lock); 335 spin_unlock(&ma_subclass_lock);
314} 336}
315EXPORT_SYMBOL(measurement_alert_subclass_unregister); 337EXPORT_SYMBOL(measurement_alert_subclass_unregister);
316
317#ifdef CONFIG_SMP
318void synchronize_irq(unsigned int irq)
319{
320 /*
321 * Not needed, the handler is protected by a lock and IRQs that occur
322 * after the handler is deleted are just NOPs.
323 */
324}
325EXPORT_SYMBOL_GPL(synchronize_irq);
326#endif
327
328#ifndef CONFIG_PCI
329
330/* Only PCI devices have dynamically-defined IRQ handlers */
331
332int request_irq(unsigned int irq, irq_handler_t handler,
333 unsigned long irqflags, const char *devname, void *dev_id)
334{
335 return -EINVAL;
336}
337EXPORT_SYMBOL_GPL(request_irq);
338
339void free_irq(unsigned int irq, void *dev_id)
340{
341 WARN_ON(1);
342}
343EXPORT_SYMBOL_GPL(free_irq);
344
345void enable_irq(unsigned int irq)
346{
347 WARN_ON(1);
348}
349EXPORT_SYMBOL_GPL(enable_irq);
350
351void disable_irq(unsigned int irq)
352{
353 WARN_ON(1);
354}
355EXPORT_SYMBOL_GPL(disable_irq);
356
357#endif /* !CONFIG_PCI */
358
359void disable_irq_nosync(unsigned int irq)
360{
361 disable_irq(irq);
362}
363EXPORT_SYMBOL_GPL(disable_irq_nosync);
364
365unsigned long probe_irq_on(void)
366{
367 return 0;
368}
369EXPORT_SYMBOL_GPL(probe_irq_on);
370
371int probe_irq_off(unsigned long val)
372{
373 return 0;
374}
375EXPORT_SYMBOL_GPL(probe_irq_off);
376
377unsigned int probe_irq_mask(unsigned long val)
378{
379 return val;
380}
381EXPORT_SYMBOL_GPL(probe_irq_mask);
diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c
index 3388b2b2a07d..adbbe7f1cb0d 100644
--- a/arch/s390/kernel/kprobes.c
+++ b/arch/s390/kernel/kprobes.c
@@ -105,14 +105,31 @@ static int __kprobes get_fixup_type(kprobe_opcode_t *insn)
105 fixup |= FIXUP_RETURN_REGISTER; 105 fixup |= FIXUP_RETURN_REGISTER;
106 break; 106 break;
107 case 0xeb: 107 case 0xeb:
108 if ((insn[2] & 0xff) == 0x44 || /* bxhg */ 108 switch (insn[2] & 0xff) {
109 (insn[2] & 0xff) == 0x45) /* bxleg */ 109 case 0x44: /* bxhg */
110 case 0x45: /* bxleg */
110 fixup = FIXUP_BRANCH_NOT_TAKEN; 111 fixup = FIXUP_BRANCH_NOT_TAKEN;
112 break;
113 }
111 break; 114 break;
112 case 0xe3: /* bctg */ 115 case 0xe3: /* bctg */
113 if ((insn[2] & 0xff) == 0x46) 116 if ((insn[2] & 0xff) == 0x46)
114 fixup = FIXUP_BRANCH_NOT_TAKEN; 117 fixup = FIXUP_BRANCH_NOT_TAKEN;
115 break; 118 break;
119 case 0xec:
120 switch (insn[2] & 0xff) {
121 case 0xe5: /* clgrb */
122 case 0xe6: /* cgrb */
123 case 0xf6: /* crb */
124 case 0xf7: /* clrb */
125 case 0xfc: /* cgib */
126 case 0xfd: /* cglib */
127 case 0xfe: /* cib */
128 case 0xff: /* clib */
129 fixup = FIXUP_BRANCH_NOT_TAKEN;
130 break;
131 }
132 break;
116 } 133 }
117 return fixup; 134 return fixup;
118} 135}
diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c
index 504175ebf8b0..c4c033819879 100644
--- a/arch/s390/kernel/nmi.c
+++ b/arch/s390/kernel/nmi.c
@@ -214,10 +214,7 @@ static int notrace s390_revalidate_registers(struct mci *mci)
214 : "0", "cc"); 214 : "0", "cc");
215#endif 215#endif
216 /* Revalidate clock comparator register */ 216 /* Revalidate clock comparator register */
217 if (S390_lowcore.clock_comparator == -1) 217 set_clock_comparator(S390_lowcore.clock_comparator);
218 set_clock_comparator(S390_lowcore.mcck_clock);
219 else
220 set_clock_comparator(S390_lowcore.clock_comparator);
221 /* Check if old PSW is valid */ 218 /* Check if old PSW is valid */
222 if (!mci->wp) 219 if (!mci->wp)
223 /* 220 /*
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 2bc3eddae34a..c5dbb335716d 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -71,6 +71,7 @@ void arch_cpu_idle(void)
71 } 71 }
72 /* Halt the cpu and keep track of cpu time accounting. */ 72 /* Halt the cpu and keep track of cpu time accounting. */
73 vtime_stop_cpu(); 73 vtime_stop_cpu();
74 local_irq_enable();
74} 75}
75 76
76void arch_cpu_idle_exit(void) 77void arch_cpu_idle_exit(void)
diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c
index e9fadb04e3c6..9556905bd3ce 100644
--- a/arch/s390/kernel/ptrace.c
+++ b/arch/s390/kernel/ptrace.c
@@ -60,11 +60,11 @@ void update_cr_regs(struct task_struct *task)
60 60
61 __ctl_store(cr, 0, 2); 61 __ctl_store(cr, 0, 2);
62 cr_new[1] = cr[1]; 62 cr_new[1] = cr[1];
63 /* Set or clear transaction execution TXC/PIFO bits 8 and 9. */ 63 /* Set or clear transaction execution TXC bit 8. */
64 if (task->thread.per_flags & PER_FLAG_NO_TE) 64 if (task->thread.per_flags & PER_FLAG_NO_TE)
65 cr_new[0] = cr[0] & ~(3UL << 54); 65 cr_new[0] = cr[0] & ~(1UL << 55);
66 else 66 else
67 cr_new[0] = cr[0] | (3UL << 54); 67 cr_new[0] = cr[0] | (1UL << 55);
68 /* Set or clear transaction execution TDC bits 62 and 63. */ 68 /* Set or clear transaction execution TDC bits 62 and 63. */
69 cr_new[2] = cr[2] & ~3UL; 69 cr_new[2] = cr[2] & ~3UL;
70 if (task->thread.per_flags & PER_FLAG_TE_ABORT_RAND) { 70 if (task->thread.per_flags & PER_FLAG_TE_ABORT_RAND) {
@@ -1299,7 +1299,7 @@ int regs_query_register_offset(const char *name)
1299 1299
1300 if (!name || *name != 'r') 1300 if (!name || *name != 'r')
1301 return -EINVAL; 1301 return -EINVAL;
1302 if (strict_strtoul(name + 1, 10, &offset)) 1302 if (kstrtoul(name + 1, 10, &offset))
1303 return -EINVAL; 1303 return -EINVAL;
1304 if (offset >= NUM_GPRS) 1304 if (offset >= NUM_GPRS)
1305 return -EINVAL; 1305 return -EINVAL;
diff --git a/arch/s390/kernel/suspend.c b/arch/s390/kernel/suspend.c
index c479d2f9605b..737bff38e3ee 100644
--- a/arch/s390/kernel/suspend.c
+++ b/arch/s390/kernel/suspend.c
@@ -10,6 +10,9 @@
10#include <linux/suspend.h> 10#include <linux/suspend.h>
11#include <linux/mm.h> 11#include <linux/mm.h>
12#include <asm/ctl_reg.h> 12#include <asm/ctl_reg.h>
13#include <asm/ipl.h>
14#include <asm/cio.h>
15#include <asm/pci.h>
13 16
14/* 17/*
15 * References to section boundaries 18 * References to section boundaries
@@ -211,3 +214,11 @@ void restore_processor_state(void)
211 __ctl_set_bit(0,28); 214 __ctl_set_bit(0,28);
212 local_mcck_enable(); 215 local_mcck_enable();
213} 216}
217
218/* Called at the end of swsusp_arch_resume */
219void s390_early_resume(void)
220{
221 lgr_info_log();
222 channel_subsystem_reinit();
223 zpci_rescan();
224}
diff --git a/arch/s390/kernel/swsusp_asm64.S b/arch/s390/kernel/swsusp_asm64.S
index c487be4cfc81..6b09fdffbd2f 100644
--- a/arch/s390/kernel/swsusp_asm64.S
+++ b/arch/s390/kernel/swsusp_asm64.S
@@ -281,11 +281,8 @@ restore_registers:
281 lghi %r2,0 281 lghi %r2,0
282 brasl %r14,arch_set_page_states 282 brasl %r14,arch_set_page_states
283 283
284 /* Log potential guest relocation */ 284 /* Call arch specific early resume code */
285 brasl %r14,lgr_info_log 285 brasl %r14,s390_early_resume
286
287 /* Reinitialize the channel subsystem */
288 brasl %r14,channel_subsystem_reinit
289 286
290 /* Return 0 */ 287 /* Return 0 */
291 lmg %r6,%r15,STACK_FRAME_OVERHEAD + __SF_GPRS(%r15) 288 lmg %r6,%r15,STACK_FRAME_OVERHEAD + __SF_GPRS(%r15)
diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c
index 876546b9cfa1..064c3082ab33 100644
--- a/arch/s390/kernel/time.c
+++ b/arch/s390/kernel/time.c
@@ -92,7 +92,6 @@ void clock_comparator_work(void)
92 struct clock_event_device *cd; 92 struct clock_event_device *cd;
93 93
94 S390_lowcore.clock_comparator = -1ULL; 94 S390_lowcore.clock_comparator = -1ULL;
95 set_clock_comparator(S390_lowcore.clock_comparator);
96 cd = &__get_cpu_var(comparators); 95 cd = &__get_cpu_var(comparators);
97 cd->event_handler(cd); 96 cd->event_handler(cd);
98} 97}
diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c
index d7776281cb60..05d75c413137 100644
--- a/arch/s390/kernel/vdso.c
+++ b/arch/s390/kernel/vdso.c
@@ -63,7 +63,7 @@ static int __init vdso_setup(char *s)
63 else if (strncmp(s, "off", 4) == 0) 63 else if (strncmp(s, "off", 4) == 0)
64 vdso_enabled = 0; 64 vdso_enabled = 0;
65 else { 65 else {
66 rc = strict_strtoul(s, 0, &val); 66 rc = kstrtoul(s, 0, &val);
67 vdso_enabled = rc ? 0 : !!val; 67 vdso_enabled = rc ? 0 : !!val;
68 } 68 }
69 return !rc; 69 return !rc;
@@ -113,11 +113,11 @@ int vdso_alloc_per_cpu(struct _lowcore *lowcore)
113 113
114 clear_table((unsigned long *) segment_table, _SEGMENT_ENTRY_EMPTY, 114 clear_table((unsigned long *) segment_table, _SEGMENT_ENTRY_EMPTY,
115 PAGE_SIZE << SEGMENT_ORDER); 115 PAGE_SIZE << SEGMENT_ORDER);
116 clear_table((unsigned long *) page_table, _PAGE_TYPE_EMPTY, 116 clear_table((unsigned long *) page_table, _PAGE_INVALID,
117 256*sizeof(unsigned long)); 117 256*sizeof(unsigned long));
118 118
119 *(unsigned long *) segment_table = _SEGMENT_ENTRY + page_table; 119 *(unsigned long *) segment_table = _SEGMENT_ENTRY + page_table;
120 *(unsigned long *) page_table = _PAGE_RO + page_frame; 120 *(unsigned long *) page_table = _PAGE_PROTECT + page_frame;
121 121
122 psal = (u32 *) (page_table + 256*sizeof(unsigned long)); 122 psal = (u32 *) (page_table + 256*sizeof(unsigned long));
123 aste = psal + 32; 123 aste = psal + 32;
diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c
index c61b9fad43cc..57c87d7d7ede 100644
--- a/arch/s390/lib/delay.c
+++ b/arch/s390/lib/delay.c
@@ -44,7 +44,6 @@ static void __udelay_disabled(unsigned long long usecs)
44 do { 44 do {
45 set_clock_comparator(end); 45 set_clock_comparator(end);
46 vtime_stop_cpu(); 46 vtime_stop_cpu();
47 local_irq_disable();
48 } while (get_tod_clock() < end); 47 } while (get_tod_clock() < end);
49 lockdep_on(); 48 lockdep_on();
50 __ctl_load(cr0, 0, 0); 49 __ctl_load(cr0, 0, 0);
@@ -64,7 +63,6 @@ static void __udelay_enabled(unsigned long long usecs)
64 set_clock_comparator(end); 63 set_clock_comparator(end);
65 } 64 }
66 vtime_stop_cpu(); 65 vtime_stop_cpu();
67 local_irq_disable();
68 if (clock_saved) 66 if (clock_saved)
69 local_tick_enable(clock_saved); 67 local_tick_enable(clock_saved);
70 } while (get_tod_clock() < end); 68 } while (get_tod_clock() < end);
diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index 50ea137a2d3c..1694d738b175 100644
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -86,28 +86,28 @@ static unsigned long follow_table(struct mm_struct *mm,
86 switch (mm->context.asce_bits & _ASCE_TYPE_MASK) { 86 switch (mm->context.asce_bits & _ASCE_TYPE_MASK) {
87 case _ASCE_TYPE_REGION1: 87 case _ASCE_TYPE_REGION1:
88 table = table + ((address >> 53) & 0x7ff); 88 table = table + ((address >> 53) & 0x7ff);
89 if (unlikely(*table & _REGION_ENTRY_INV)) 89 if (unlikely(*table & _REGION_ENTRY_INVALID))
90 return -0x39UL; 90 return -0x39UL;
91 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 91 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
92 /* fallthrough */ 92 /* fallthrough */
93 case _ASCE_TYPE_REGION2: 93 case _ASCE_TYPE_REGION2:
94 table = table + ((address >> 42) & 0x7ff); 94 table = table + ((address >> 42) & 0x7ff);
95 if (unlikely(*table & _REGION_ENTRY_INV)) 95 if (unlikely(*table & _REGION_ENTRY_INVALID))
96 return -0x3aUL; 96 return -0x3aUL;
97 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 97 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
98 /* fallthrough */ 98 /* fallthrough */
99 case _ASCE_TYPE_REGION3: 99 case _ASCE_TYPE_REGION3:
100 table = table + ((address >> 31) & 0x7ff); 100 table = table + ((address >> 31) & 0x7ff);
101 if (unlikely(*table & _REGION_ENTRY_INV)) 101 if (unlikely(*table & _REGION_ENTRY_INVALID))
102 return -0x3bUL; 102 return -0x3bUL;
103 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 103 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
104 /* fallthrough */ 104 /* fallthrough */
105 case _ASCE_TYPE_SEGMENT: 105 case _ASCE_TYPE_SEGMENT:
106 table = table + ((address >> 20) & 0x7ff); 106 table = table + ((address >> 20) & 0x7ff);
107 if (unlikely(*table & _SEGMENT_ENTRY_INV)) 107 if (unlikely(*table & _SEGMENT_ENTRY_INVALID))
108 return -0x10UL; 108 return -0x10UL;
109 if (unlikely(*table & _SEGMENT_ENTRY_LARGE)) { 109 if (unlikely(*table & _SEGMENT_ENTRY_LARGE)) {
110 if (write && (*table & _SEGMENT_ENTRY_RO)) 110 if (write && (*table & _SEGMENT_ENTRY_PROTECT))
111 return -0x04UL; 111 return -0x04UL;
112 return (*table & _SEGMENT_ENTRY_ORIGIN_LARGE) + 112 return (*table & _SEGMENT_ENTRY_ORIGIN_LARGE) +
113 (address & ~_SEGMENT_ENTRY_ORIGIN_LARGE); 113 (address & ~_SEGMENT_ENTRY_ORIGIN_LARGE);
@@ -117,7 +117,7 @@ static unsigned long follow_table(struct mm_struct *mm,
117 table = table + ((address >> 12) & 0xff); 117 table = table + ((address >> 12) & 0xff);
118 if (unlikely(*table & _PAGE_INVALID)) 118 if (unlikely(*table & _PAGE_INVALID))
119 return -0x11UL; 119 return -0x11UL;
120 if (write && (*table & _PAGE_RO)) 120 if (write && (*table & _PAGE_PROTECT))
121 return -0x04UL; 121 return -0x04UL;
122 return (*table & PAGE_MASK) + (address & ~PAGE_MASK); 122 return (*table & PAGE_MASK) + (address & ~PAGE_MASK);
123} 123}
@@ -130,13 +130,13 @@ static unsigned long follow_table(struct mm_struct *mm,
130 unsigned long *table = (unsigned long *)__pa(mm->pgd); 130 unsigned long *table = (unsigned long *)__pa(mm->pgd);
131 131
132 table = table + ((address >> 20) & 0x7ff); 132 table = table + ((address >> 20) & 0x7ff);
133 if (unlikely(*table & _SEGMENT_ENTRY_INV)) 133 if (unlikely(*table & _SEGMENT_ENTRY_INVALID))
134 return -0x10UL; 134 return -0x10UL;
135 table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN); 135 table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN);
136 table = table + ((address >> 12) & 0xff); 136 table = table + ((address >> 12) & 0xff);
137 if (unlikely(*table & _PAGE_INVALID)) 137 if (unlikely(*table & _PAGE_INVALID))
138 return -0x11UL; 138 return -0x11UL;
139 if (write && (*table & _PAGE_RO)) 139 if (write && (*table & _PAGE_PROTECT))
140 return -0x04UL; 140 return -0x04UL;
141 return (*table & PAGE_MASK) + (address & ~PAGE_MASK); 141 return (*table & PAGE_MASK) + (address & ~PAGE_MASK);
142} 142}
diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c
index 3ad65b04ac15..46d517c3c763 100644
--- a/arch/s390/mm/dump_pagetables.c
+++ b/arch/s390/mm/dump_pagetables.c
@@ -53,7 +53,7 @@ static void print_prot(struct seq_file *m, unsigned int pr, int level)
53 seq_printf(m, "I\n"); 53 seq_printf(m, "I\n");
54 return; 54 return;
55 } 55 }
56 seq_printf(m, "%s", pr & _PAGE_RO ? "RO " : "RW "); 56 seq_printf(m, "%s", pr & _PAGE_PROTECT ? "RO " : "RW ");
57 seq_printf(m, "%s", pr & _PAGE_CO ? "CO " : " "); 57 seq_printf(m, "%s", pr & _PAGE_CO ? "CO " : " ");
58 seq_putc(m, '\n'); 58 seq_putc(m, '\n');
59} 59}
@@ -105,12 +105,12 @@ static void note_page(struct seq_file *m, struct pg_state *st,
105} 105}
106 106
107/* 107/*
108 * The actual page table walker functions. In order to keep the implementation 108 * The actual page table walker functions. In order to keep the
109 * of print_prot() short, we only check and pass _PAGE_INVALID and _PAGE_RO 109 * implementation of print_prot() short, we only check and pass
110 * flags to note_page() if a region, segment or page table entry is invalid or 110 * _PAGE_INVALID and _PAGE_PROTECT flags to note_page() if a region,
111 * read-only. 111 * segment or page table entry is invalid or read-only.
112 * After all it's just a hint that the current level being walked contains an 112 * After all it's just a hint that the current level being walked
113 * invalid or read-only entry. 113 * contains an invalid or read-only entry.
114 */ 114 */
115static void walk_pte_level(struct seq_file *m, struct pg_state *st, 115static void walk_pte_level(struct seq_file *m, struct pg_state *st,
116 pmd_t *pmd, unsigned long addr) 116 pmd_t *pmd, unsigned long addr)
@@ -122,14 +122,14 @@ static void walk_pte_level(struct seq_file *m, struct pg_state *st,
122 for (i = 0; i < PTRS_PER_PTE && addr < max_addr; i++) { 122 for (i = 0; i < PTRS_PER_PTE && addr < max_addr; i++) {
123 st->current_address = addr; 123 st->current_address = addr;
124 pte = pte_offset_kernel(pmd, addr); 124 pte = pte_offset_kernel(pmd, addr);
125 prot = pte_val(*pte) & (_PAGE_RO | _PAGE_INVALID); 125 prot = pte_val(*pte) & (_PAGE_PROTECT | _PAGE_INVALID);
126 note_page(m, st, prot, 4); 126 note_page(m, st, prot, 4);
127 addr += PAGE_SIZE; 127 addr += PAGE_SIZE;
128 } 128 }
129} 129}
130 130
131#ifdef CONFIG_64BIT 131#ifdef CONFIG_64BIT
132#define _PMD_PROT_MASK (_SEGMENT_ENTRY_RO | _SEGMENT_ENTRY_CO) 132#define _PMD_PROT_MASK (_SEGMENT_ENTRY_PROTECT | _SEGMENT_ENTRY_CO)
133#else 133#else
134#define _PMD_PROT_MASK 0 134#define _PMD_PROT_MASK 0
135#endif 135#endif
diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c
index 1f5315d1215c..5d758db27bdc 100644
--- a/arch/s390/mm/gup.c
+++ b/arch/s390/mm/gup.c
@@ -24,7 +24,7 @@ static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
24 pte_t *ptep, pte; 24 pte_t *ptep, pte;
25 struct page *page; 25 struct page *page;
26 26
27 mask = (write ? _PAGE_RO : 0) | _PAGE_INVALID | _PAGE_SPECIAL; 27 mask = (write ? _PAGE_PROTECT : 0) | _PAGE_INVALID | _PAGE_SPECIAL;
28 28
29 ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr); 29 ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr);
30 do { 30 do {
@@ -55,8 +55,8 @@ static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
55 struct page *head, *page, *tail; 55 struct page *head, *page, *tail;
56 int refs; 56 int refs;
57 57
58 result = write ? 0 : _SEGMENT_ENTRY_RO; 58 result = write ? 0 : _SEGMENT_ENTRY_PROTECT;
59 mask = result | _SEGMENT_ENTRY_INV; 59 mask = result | _SEGMENT_ENTRY_INVALID;
60 if ((pmd_val(pmd) & mask) != result) 60 if ((pmd_val(pmd) & mask) != result)
61 return 0; 61 return 0;
62 VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT)); 62 VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT));
diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c
index 121089d57802..248445f92604 100644
--- a/arch/s390/mm/hugetlbpage.c
+++ b/arch/s390/mm/hugetlbpage.c
@@ -8,21 +8,127 @@
8#include <linux/mm.h> 8#include <linux/mm.h>
9#include <linux/hugetlb.h> 9#include <linux/hugetlb.h>
10 10
11static inline pmd_t __pte_to_pmd(pte_t pte)
12{
13 int none, young, prot;
14 pmd_t pmd;
15
16 /*
17 * Convert encoding pte bits pmd bits
18 * .IR...wrdytp ..R...I...y.
19 * empty .10...000000 -> ..0...1...0.
20 * prot-none, clean, old .11...000001 -> ..0...1...1.
21 * prot-none, clean, young .11...000101 -> ..1...1...1.
22 * prot-none, dirty, old .10...001001 -> ..0...1...1.
23 * prot-none, dirty, young .10...001101 -> ..1...1...1.
24 * read-only, clean, old .11...010001 -> ..1...1...0.
25 * read-only, clean, young .01...010101 -> ..1...0...1.
26 * read-only, dirty, old .11...011001 -> ..1...1...0.
27 * read-only, dirty, young .01...011101 -> ..1...0...1.
28 * read-write, clean, old .11...110001 -> ..0...1...0.
29 * read-write, clean, young .01...110101 -> ..0...0...1.
30 * read-write, dirty, old .10...111001 -> ..0...1...0.
31 * read-write, dirty, young .00...111101 -> ..0...0...1.
32 * Huge ptes are dirty by definition, a clean pte is made dirty
33 * by the conversion.
34 */
35 if (pte_present(pte)) {
36 pmd_val(pmd) = pte_val(pte) & PAGE_MASK;
37 if (pte_val(pte) & _PAGE_INVALID)
38 pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID;
39 none = (pte_val(pte) & _PAGE_PRESENT) &&
40 !(pte_val(pte) & _PAGE_READ) &&
41 !(pte_val(pte) & _PAGE_WRITE);
42 prot = (pte_val(pte) & _PAGE_PROTECT) &&
43 !(pte_val(pte) & _PAGE_WRITE);
44 young = pte_val(pte) & _PAGE_YOUNG;
45 if (none || young)
46 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG;
47 if (prot || (none && young))
48 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT;
49 } else
50 pmd_val(pmd) = _SEGMENT_ENTRY_INVALID;
51 return pmd;
52}
53
54static inline pte_t __pmd_to_pte(pmd_t pmd)
55{
56 pte_t pte;
57
58 /*
59 * Convert encoding pmd bits pte bits
60 * ..R...I...y. .IR...wrdytp
61 * empty ..0...1...0. -> .10...000000
62 * prot-none, old ..0...1...1. -> .10...001001
63 * prot-none, young ..1...1...1. -> .10...001101
64 * read-only, old ..1...1...0. -> .11...011001
65 * read-only, young ..1...0...1. -> .01...011101
66 * read-write, old ..0...1...0. -> .10...111001
67 * read-write, young ..0...0...1. -> .00...111101
68 * Huge ptes are dirty by definition
69 */
70 if (pmd_present(pmd)) {
71 pte_val(pte) = _PAGE_PRESENT | _PAGE_LARGE | _PAGE_DIRTY |
72 (pmd_val(pmd) & PAGE_MASK);
73 if (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID)
74 pte_val(pte) |= _PAGE_INVALID;
75 if (pmd_prot_none(pmd)) {
76 if (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT)
77 pte_val(pte) |= _PAGE_YOUNG;
78 } else {
79 pte_val(pte) |= _PAGE_READ;
80 if (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT)
81 pte_val(pte) |= _PAGE_PROTECT;
82 else
83 pte_val(pte) |= _PAGE_WRITE;
84 if (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG)
85 pte_val(pte) |= _PAGE_YOUNG;
86 }
87 } else
88 pte_val(pte) = _PAGE_INVALID;
89 return pte;
90}
11 91
12void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 92void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
13 pte_t *pteptr, pte_t pteval) 93 pte_t *ptep, pte_t pte)
14{ 94{
15 pmd_t *pmdp = (pmd_t *) pteptr; 95 pmd_t pmd;
16 unsigned long mask;
17 96
97 pmd = __pte_to_pmd(pte);
18 if (!MACHINE_HAS_HPAGE) { 98 if (!MACHINE_HAS_HPAGE) {
19 pteptr = (pte_t *) pte_page(pteval)[1].index; 99 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
20 mask = pte_val(pteval) & 100 pmd_val(pmd) |= pte_page(pte)[1].index;
21 (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO); 101 } else
22 pte_val(pteval) = (_SEGMENT_ENTRY + __pa(pteptr)) | mask; 102 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO;
103 *(pmd_t *) ptep = pmd;
104}
105
106pte_t huge_ptep_get(pte_t *ptep)
107{
108 unsigned long origin;
109 pmd_t pmd;
110
111 pmd = *(pmd_t *) ptep;
112 if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) {
113 origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN;
114 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
115 pmd_val(pmd) |= *(unsigned long *) origin;
23 } 116 }
117 return __pmd_to_pte(pmd);
118}
24 119
25 pmd_val(*pmdp) = pte_val(pteval); 120pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
121 unsigned long addr, pte_t *ptep)
122{
123 pmd_t *pmdp = (pmd_t *) ptep;
124 pte_t pte = huge_ptep_get(ptep);
125
126 if (MACHINE_HAS_IDTE)
127 __pmd_idte(addr, pmdp);
128 else
129 __pmd_csp(pmdp);
130 pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
131 return pte;
26} 132}
27 133
28int arch_prepare_hugepage(struct page *page) 134int arch_prepare_hugepage(struct page *page)
@@ -58,7 +164,7 @@ void arch_release_hugepage(struct page *page)
58 ptep = (pte_t *) page[1].index; 164 ptep = (pte_t *) page[1].index;
59 if (!ptep) 165 if (!ptep)
60 return; 166 return;
61 clear_table((unsigned long *) ptep, _PAGE_TYPE_EMPTY, 167 clear_table((unsigned long *) ptep, _PAGE_INVALID,
62 PTRS_PER_PTE * sizeof(pte_t)); 168 PTRS_PER_PTE * sizeof(pte_t));
63 page_table_free(&init_mm, (unsigned long *) ptep); 169 page_table_free(&init_mm, (unsigned long *) ptep);
64 page[1].index = 0; 170 page[1].index = 0;
diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c
index 80adfbf75065..990397420e6b 100644
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -118,7 +118,7 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
118 pte = pte_offset_kernel(pmd, address); 118 pte = pte_offset_kernel(pmd, address);
119 if (!enable) { 119 if (!enable) {
120 __ptep_ipte(address, pte); 120 __ptep_ipte(address, pte);
121 pte_val(*pte) = _PAGE_TYPE_EMPTY; 121 pte_val(*pte) = _PAGE_INVALID;
122 continue; 122 continue;
123 } 123 }
124 pte_val(*pte) = __pa(address); 124 pte_val(*pte) = __pa(address);
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index a8154a1a2c94..6d16132d0850 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -161,7 +161,7 @@ static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table)
161 struct gmap_rmap *rmap; 161 struct gmap_rmap *rmap;
162 struct page *page; 162 struct page *page;
163 163
164 if (*table & _SEGMENT_ENTRY_INV) 164 if (*table & _SEGMENT_ENTRY_INVALID)
165 return 0; 165 return 0;
166 page = pfn_to_page(*table >> PAGE_SHIFT); 166 page = pfn_to_page(*table >> PAGE_SHIFT);
167 mp = (struct gmap_pgtable *) page->index; 167 mp = (struct gmap_pgtable *) page->index;
@@ -172,7 +172,7 @@ static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table)
172 kfree(rmap); 172 kfree(rmap);
173 break; 173 break;
174 } 174 }
175 *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr; 175 *table = mp->vmaddr | _SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_PROTECT;
176 return 1; 176 return 1;
177} 177}
178 178
@@ -258,7 +258,7 @@ static int gmap_alloc_table(struct gmap *gmap,
258 return -ENOMEM; 258 return -ENOMEM;
259 new = (unsigned long *) page_to_phys(page); 259 new = (unsigned long *) page_to_phys(page);
260 crst_table_init(new, init); 260 crst_table_init(new, init);
261 if (*table & _REGION_ENTRY_INV) { 261 if (*table & _REGION_ENTRY_INVALID) {
262 list_add(&page->lru, &gmap->crst_list); 262 list_add(&page->lru, &gmap->crst_list);
263 *table = (unsigned long) new | _REGION_ENTRY_LENGTH | 263 *table = (unsigned long) new | _REGION_ENTRY_LENGTH |
264 (*table & _REGION_ENTRY_TYPE_MASK); 264 (*table & _REGION_ENTRY_TYPE_MASK);
@@ -292,22 +292,22 @@ int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len)
292 for (off = 0; off < len; off += PMD_SIZE) { 292 for (off = 0; off < len; off += PMD_SIZE) {
293 /* Walk the guest addr space page table */ 293 /* Walk the guest addr space page table */
294 table = gmap->table + (((to + off) >> 53) & 0x7ff); 294 table = gmap->table + (((to + off) >> 53) & 0x7ff);
295 if (*table & _REGION_ENTRY_INV) 295 if (*table & _REGION_ENTRY_INVALID)
296 goto out; 296 goto out;
297 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 297 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
298 table = table + (((to + off) >> 42) & 0x7ff); 298 table = table + (((to + off) >> 42) & 0x7ff);
299 if (*table & _REGION_ENTRY_INV) 299 if (*table & _REGION_ENTRY_INVALID)
300 goto out; 300 goto out;
301 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 301 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
302 table = table + (((to + off) >> 31) & 0x7ff); 302 table = table + (((to + off) >> 31) & 0x7ff);
303 if (*table & _REGION_ENTRY_INV) 303 if (*table & _REGION_ENTRY_INVALID)
304 goto out; 304 goto out;
305 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 305 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
306 table = table + (((to + off) >> 20) & 0x7ff); 306 table = table + (((to + off) >> 20) & 0x7ff);
307 307
308 /* Clear segment table entry in guest address space. */ 308 /* Clear segment table entry in guest address space. */
309 flush |= gmap_unlink_segment(gmap, table); 309 flush |= gmap_unlink_segment(gmap, table);
310 *table = _SEGMENT_ENTRY_INV; 310 *table = _SEGMENT_ENTRY_INVALID;
311 } 311 }
312out: 312out:
313 spin_unlock(&gmap->mm->page_table_lock); 313 spin_unlock(&gmap->mm->page_table_lock);
@@ -345,17 +345,17 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from,
345 for (off = 0; off < len; off += PMD_SIZE) { 345 for (off = 0; off < len; off += PMD_SIZE) {
346 /* Walk the gmap address space page table */ 346 /* Walk the gmap address space page table */
347 table = gmap->table + (((to + off) >> 53) & 0x7ff); 347 table = gmap->table + (((to + off) >> 53) & 0x7ff);
348 if ((*table & _REGION_ENTRY_INV) && 348 if ((*table & _REGION_ENTRY_INVALID) &&
349 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY)) 349 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY))
350 goto out_unmap; 350 goto out_unmap;
351 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 351 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
352 table = table + (((to + off) >> 42) & 0x7ff); 352 table = table + (((to + off) >> 42) & 0x7ff);
353 if ((*table & _REGION_ENTRY_INV) && 353 if ((*table & _REGION_ENTRY_INVALID) &&
354 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY)) 354 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY))
355 goto out_unmap; 355 goto out_unmap;
356 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 356 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
357 table = table + (((to + off) >> 31) & 0x7ff); 357 table = table + (((to + off) >> 31) & 0x7ff);
358 if ((*table & _REGION_ENTRY_INV) && 358 if ((*table & _REGION_ENTRY_INVALID) &&
359 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY)) 359 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY))
360 goto out_unmap; 360 goto out_unmap;
361 table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN); 361 table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN);
@@ -363,7 +363,8 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from,
363 363
364 /* Store 'from' address in an invalid segment table entry. */ 364 /* Store 'from' address in an invalid segment table entry. */
365 flush |= gmap_unlink_segment(gmap, table); 365 flush |= gmap_unlink_segment(gmap, table);
366 *table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | (from + off); 366 *table = (from + off) | (_SEGMENT_ENTRY_INVALID |
367 _SEGMENT_ENTRY_PROTECT);
367 } 368 }
368 spin_unlock(&gmap->mm->page_table_lock); 369 spin_unlock(&gmap->mm->page_table_lock);
369 up_read(&gmap->mm->mmap_sem); 370 up_read(&gmap->mm->mmap_sem);
@@ -384,15 +385,15 @@ static unsigned long *gmap_table_walk(unsigned long address, struct gmap *gmap)
384 unsigned long *table; 385 unsigned long *table;
385 386
386 table = gmap->table + ((address >> 53) & 0x7ff); 387 table = gmap->table + ((address >> 53) & 0x7ff);
387 if (unlikely(*table & _REGION_ENTRY_INV)) 388 if (unlikely(*table & _REGION_ENTRY_INVALID))
388 return ERR_PTR(-EFAULT); 389 return ERR_PTR(-EFAULT);
389 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 390 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
390 table = table + ((address >> 42) & 0x7ff); 391 table = table + ((address >> 42) & 0x7ff);
391 if (unlikely(*table & _REGION_ENTRY_INV)) 392 if (unlikely(*table & _REGION_ENTRY_INVALID))
392 return ERR_PTR(-EFAULT); 393 return ERR_PTR(-EFAULT);
393 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 394 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
394 table = table + ((address >> 31) & 0x7ff); 395 table = table + ((address >> 31) & 0x7ff);
395 if (unlikely(*table & _REGION_ENTRY_INV)) 396 if (unlikely(*table & _REGION_ENTRY_INVALID))
396 return ERR_PTR(-EFAULT); 397 return ERR_PTR(-EFAULT);
397 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 398 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
398 table = table + ((address >> 20) & 0x7ff); 399 table = table + ((address >> 20) & 0x7ff);
@@ -422,11 +423,11 @@ unsigned long __gmap_translate(unsigned long address, struct gmap *gmap)
422 return PTR_ERR(segment_ptr); 423 return PTR_ERR(segment_ptr);
423 /* Convert the gmap address to an mm address. */ 424 /* Convert the gmap address to an mm address. */
424 segment = *segment_ptr; 425 segment = *segment_ptr;
425 if (!(segment & _SEGMENT_ENTRY_INV)) { 426 if (!(segment & _SEGMENT_ENTRY_INVALID)) {
426 page = pfn_to_page(segment >> PAGE_SHIFT); 427 page = pfn_to_page(segment >> PAGE_SHIFT);
427 mp = (struct gmap_pgtable *) page->index; 428 mp = (struct gmap_pgtable *) page->index;
428 return mp->vmaddr | (address & ~PMD_MASK); 429 return mp->vmaddr | (address & ~PMD_MASK);
429 } else if (segment & _SEGMENT_ENTRY_RO) { 430 } else if (segment & _SEGMENT_ENTRY_PROTECT) {
430 vmaddr = segment & _SEGMENT_ENTRY_ORIGIN; 431 vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
431 return vmaddr | (address & ~PMD_MASK); 432 return vmaddr | (address & ~PMD_MASK);
432 } 433 }
@@ -517,8 +518,8 @@ static void gmap_disconnect_pgtable(struct mm_struct *mm, unsigned long *table)
517 page = pfn_to_page(__pa(table) >> PAGE_SHIFT); 518 page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
518 mp = (struct gmap_pgtable *) page->index; 519 mp = (struct gmap_pgtable *) page->index;
519 list_for_each_entry_safe(rmap, next, &mp->mapper, list) { 520 list_for_each_entry_safe(rmap, next, &mp->mapper, list) {
520 *rmap->entry = 521 *rmap->entry = mp->vmaddr | (_SEGMENT_ENTRY_INVALID |
521 _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr; 522 _SEGMENT_ENTRY_PROTECT);
522 list_del(&rmap->list); 523 list_del(&rmap->list);
523 kfree(rmap); 524 kfree(rmap);
524 flush = 1; 525 flush = 1;
@@ -545,13 +546,13 @@ unsigned long __gmap_fault(unsigned long address, struct gmap *gmap)
545 /* Convert the gmap address to an mm address. */ 546 /* Convert the gmap address to an mm address. */
546 while (1) { 547 while (1) {
547 segment = *segment_ptr; 548 segment = *segment_ptr;
548 if (!(segment & _SEGMENT_ENTRY_INV)) { 549 if (!(segment & _SEGMENT_ENTRY_INVALID)) {
549 /* Page table is present */ 550 /* Page table is present */
550 page = pfn_to_page(segment >> PAGE_SHIFT); 551 page = pfn_to_page(segment >> PAGE_SHIFT);
551 mp = (struct gmap_pgtable *) page->index; 552 mp = (struct gmap_pgtable *) page->index;
552 return mp->vmaddr | (address & ~PMD_MASK); 553 return mp->vmaddr | (address & ~PMD_MASK);
553 } 554 }
554 if (!(segment & _SEGMENT_ENTRY_RO)) 555 if (!(segment & _SEGMENT_ENTRY_PROTECT))
555 /* Nothing mapped in the gmap address space. */ 556 /* Nothing mapped in the gmap address space. */
556 break; 557 break;
557 rc = gmap_connect_pgtable(address, segment, segment_ptr, gmap); 558 rc = gmap_connect_pgtable(address, segment, segment_ptr, gmap);
@@ -586,25 +587,25 @@ void gmap_discard(unsigned long from, unsigned long to, struct gmap *gmap)
586 while (address < to) { 587 while (address < to) {
587 /* Walk the gmap address space page table */ 588 /* Walk the gmap address space page table */
588 table = gmap->table + ((address >> 53) & 0x7ff); 589 table = gmap->table + ((address >> 53) & 0x7ff);
589 if (unlikely(*table & _REGION_ENTRY_INV)) { 590 if (unlikely(*table & _REGION_ENTRY_INVALID)) {
590 address = (address + PMD_SIZE) & PMD_MASK; 591 address = (address + PMD_SIZE) & PMD_MASK;
591 continue; 592 continue;
592 } 593 }
593 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 594 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
594 table = table + ((address >> 42) & 0x7ff); 595 table = table + ((address >> 42) & 0x7ff);
595 if (unlikely(*table & _REGION_ENTRY_INV)) { 596 if (unlikely(*table & _REGION_ENTRY_INVALID)) {
596 address = (address + PMD_SIZE) & PMD_MASK; 597 address = (address + PMD_SIZE) & PMD_MASK;
597 continue; 598 continue;
598 } 599 }
599 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 600 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
600 table = table + ((address >> 31) & 0x7ff); 601 table = table + ((address >> 31) & 0x7ff);
601 if (unlikely(*table & _REGION_ENTRY_INV)) { 602 if (unlikely(*table & _REGION_ENTRY_INVALID)) {
602 address = (address + PMD_SIZE) & PMD_MASK; 603 address = (address + PMD_SIZE) & PMD_MASK;
603 continue; 604 continue;
604 } 605 }
605 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); 606 table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
606 table = table + ((address >> 20) & 0x7ff); 607 table = table + ((address >> 20) & 0x7ff);
607 if (unlikely(*table & _SEGMENT_ENTRY_INV)) { 608 if (unlikely(*table & _SEGMENT_ENTRY_INVALID)) {
608 address = (address + PMD_SIZE) & PMD_MASK; 609 address = (address + PMD_SIZE) & PMD_MASK;
609 continue; 610 continue;
610 } 611 }
@@ -687,7 +688,7 @@ int gmap_ipte_notify(struct gmap *gmap, unsigned long start, unsigned long len)
687 continue; 688 continue;
688 /* Set notification bit in the pgste of the pte */ 689 /* Set notification bit in the pgste of the pte */
689 entry = *ptep; 690 entry = *ptep;
690 if ((pte_val(entry) & (_PAGE_INVALID | _PAGE_RO)) == 0) { 691 if ((pte_val(entry) & (_PAGE_INVALID | _PAGE_PROTECT)) == 0) {
691 pgste = pgste_get_lock(ptep); 692 pgste = pgste_get_lock(ptep);
692 pgste_val(pgste) |= PGSTE_IN_BIT; 693 pgste_val(pgste) |= PGSTE_IN_BIT;
693 pgste_set_unlock(ptep, pgste); 694 pgste_set_unlock(ptep, pgste);
@@ -752,8 +753,9 @@ static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
752 page->index = (unsigned long) mp; 753 page->index = (unsigned long) mp;
753 atomic_set(&page->_mapcount, 3); 754 atomic_set(&page->_mapcount, 3);
754 table = (unsigned long *) page_to_phys(page); 755 table = (unsigned long *) page_to_phys(page);
755 clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2); 756 clear_table(table, _PAGE_INVALID, PAGE_SIZE/2);
756 clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2); 757 clear_table(table + PTRS_PER_PTE, PGSTE_HR_BIT | PGSTE_HC_BIT,
758 PAGE_SIZE/2);
757 return table; 759 return table;
758} 760}
759 761
@@ -791,26 +793,21 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
791 pgste_val(new) |= (key & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; 793 pgste_val(new) |= (key & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
792 pgste_val(new) |= (key & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; 794 pgste_val(new) |= (key & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
793 if (!(pte_val(*ptep) & _PAGE_INVALID)) { 795 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
794 unsigned long address, bits; 796 unsigned long address, bits, skey;
795 unsigned char skey;
796 797
797 address = pte_val(*ptep) & PAGE_MASK; 798 address = pte_val(*ptep) & PAGE_MASK;
798 skey = page_get_storage_key(address); 799 skey = (unsigned long) page_get_storage_key(address);
799 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); 800 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
801 skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
800 /* Set storage key ACC and FP */ 802 /* Set storage key ACC and FP */
801 page_set_storage_key(address, 803 page_set_storage_key(address, skey, !nq);
802 (key & (_PAGE_ACC_BITS | _PAGE_FP_BIT)),
803 !nq);
804
805 /* Merge host changed & referenced into pgste */ 804 /* Merge host changed & referenced into pgste */
806 pgste_val(new) |= bits << 52; 805 pgste_val(new) |= bits << 52;
807 /* Transfer skey changed & referenced bit to kvm user bits */
808 pgste_val(new) |= bits << 45; /* PGSTE_UR_BIT & PGSTE_UC_BIT */
809 } 806 }
810 /* changing the guest storage key is considered a change of the page */ 807 /* changing the guest storage key is considered a change of the page */
811 if ((pgste_val(new) ^ pgste_val(old)) & 808 if ((pgste_val(new) ^ pgste_val(old)) &
812 (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) 809 (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
813 pgste_val(new) |= PGSTE_UC_BIT; 810 pgste_val(new) |= PGSTE_HC_BIT;
814 811
815 pgste_set_unlock(ptep, new); 812 pgste_set_unlock(ptep, new);
816 pte_unmap_unlock(*ptep, ptl); 813 pte_unmap_unlock(*ptep, ptl);
@@ -878,7 +875,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr)
878 pgtable_page_ctor(page); 875 pgtable_page_ctor(page);
879 atomic_set(&page->_mapcount, 1); 876 atomic_set(&page->_mapcount, 1);
880 table = (unsigned long *) page_to_phys(page); 877 table = (unsigned long *) page_to_phys(page);
881 clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE); 878 clear_table(table, _PAGE_INVALID, PAGE_SIZE);
882 spin_lock_bh(&mm->context.list_lock); 879 spin_lock_bh(&mm->context.list_lock);
883 list_add(&page->lru, &mm->context.pgtable_list); 880 list_add(&page->lru, &mm->context.pgtable_list);
884 } else { 881 } else {
@@ -1007,7 +1004,6 @@ void tlb_table_flush(struct mmu_gather *tlb)
1007 struct mmu_table_batch **batch = &tlb->batch; 1004 struct mmu_table_batch **batch = &tlb->batch;
1008 1005
1009 if (*batch) { 1006 if (*batch) {
1010 __tlb_flush_mm(tlb->mm);
1011 call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu); 1007 call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
1012 *batch = NULL; 1008 *batch = NULL;
1013 } 1009 }
@@ -1017,11 +1013,12 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
1017{ 1013{
1018 struct mmu_table_batch **batch = &tlb->batch; 1014 struct mmu_table_batch **batch = &tlb->batch;
1019 1015
1016 tlb->mm->context.flush_mm = 1;
1020 if (*batch == NULL) { 1017 if (*batch == NULL) {
1021 *batch = (struct mmu_table_batch *) 1018 *batch = (struct mmu_table_batch *)
1022 __get_free_page(GFP_NOWAIT | __GFP_NOWARN); 1019 __get_free_page(GFP_NOWAIT | __GFP_NOWARN);
1023 if (*batch == NULL) { 1020 if (*batch == NULL) {
1024 __tlb_flush_mm(tlb->mm); 1021 __tlb_flush_mm_lazy(tlb->mm);
1025 tlb_remove_table_one(table); 1022 tlb_remove_table_one(table);
1026 return; 1023 return;
1027 } 1024 }
@@ -1029,7 +1026,7 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
1029 } 1026 }
1030 (*batch)->tables[(*batch)->nr++] = table; 1027 (*batch)->tables[(*batch)->nr++] = table;
1031 if ((*batch)->nr == MAX_TABLE_BATCH) 1028 if ((*batch)->nr == MAX_TABLE_BATCH)
1032 tlb_table_flush(tlb); 1029 tlb_flush_mmu(tlb);
1033} 1030}
1034 1031
1035#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1032#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -1198,9 +1195,9 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
1198 list_del(lh); 1195 list_del(lh);
1199 } 1196 }
1200 ptep = (pte_t *) pgtable; 1197 ptep = (pte_t *) pgtable;
1201 pte_val(*ptep) = _PAGE_TYPE_EMPTY; 1198 pte_val(*ptep) = _PAGE_INVALID;
1202 ptep++; 1199 ptep++;
1203 pte_val(*ptep) = _PAGE_TYPE_EMPTY; 1200 pte_val(*ptep) = _PAGE_INVALID;
1204 return pgtable; 1201 return pgtable;
1205} 1202}
1206#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1203#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index 8b268fcc4612..bcfb70b60be6 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -69,7 +69,7 @@ static pte_t __ref *vmem_pte_alloc(unsigned long address)
69 pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); 69 pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t));
70 if (!pte) 70 if (!pte)
71 return NULL; 71 return NULL;
72 clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, 72 clear_table((unsigned long *) pte, _PAGE_INVALID,
73 PTRS_PER_PTE * sizeof(pte_t)); 73 PTRS_PER_PTE * sizeof(pte_t));
74 return pte; 74 return pte;
75} 75}
@@ -101,7 +101,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
101 !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) { 101 !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) {
102 pud_val(*pu_dir) = __pa(address) | 102 pud_val(*pu_dir) = __pa(address) |
103 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE | 103 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE |
104 (ro ? _REGION_ENTRY_RO : 0); 104 (ro ? _REGION_ENTRY_PROTECT : 0);
105 address += PUD_SIZE; 105 address += PUD_SIZE;
106 continue; 106 continue;
107 } 107 }
@@ -118,7 +118,8 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
118 !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) { 118 !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) {
119 pmd_val(*pm_dir) = __pa(address) | 119 pmd_val(*pm_dir) = __pa(address) |
120 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | 120 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
121 (ro ? _SEGMENT_ENTRY_RO : 0); 121 _SEGMENT_ENTRY_YOUNG |
122 (ro ? _SEGMENT_ENTRY_PROTECT : 0);
122 address += PMD_SIZE; 123 address += PMD_SIZE;
123 continue; 124 continue;
124 } 125 }
@@ -131,7 +132,8 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
131 } 132 }
132 133
133 pt_dir = pte_offset_kernel(pm_dir, address); 134 pt_dir = pte_offset_kernel(pm_dir, address);
134 pte_val(*pt_dir) = __pa(address) | (ro ? _PAGE_RO : 0); 135 pte_val(*pt_dir) = __pa(address) |
136 pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
135 address += PAGE_SIZE; 137 address += PAGE_SIZE;
136 } 138 }
137 ret = 0; 139 ret = 0;
@@ -154,7 +156,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
154 pte_t *pt_dir; 156 pte_t *pt_dir;
155 pte_t pte; 157 pte_t pte;
156 158
157 pte_val(pte) = _PAGE_TYPE_EMPTY; 159 pte_val(pte) = _PAGE_INVALID;
158 while (address < end) { 160 while (address < end) {
159 pg_dir = pgd_offset_k(address); 161 pg_dir = pgd_offset_k(address);
160 if (pgd_none(*pg_dir)) { 162 if (pgd_none(*pg_dir)) {
@@ -255,7 +257,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
255 new_page =__pa(vmem_alloc_pages(0)); 257 new_page =__pa(vmem_alloc_pages(0));
256 if (!new_page) 258 if (!new_page)
257 goto out; 259 goto out;
258 pte_val(*pt_dir) = __pa(new_page); 260 pte_val(*pt_dir) =
261 __pa(new_page) | pgprot_val(PAGE_KERNEL);
259 } 262 }
260 address += PAGE_SIZE; 263 address += PAGE_SIZE;
261 } 264 }
diff --git a/arch/s390/pci/Makefile b/arch/s390/pci/Makefile
index 086a2e37935d..a9e1dc4ae442 100644
--- a/arch/s390/pci/Makefile
+++ b/arch/s390/pci/Makefile
@@ -2,5 +2,5 @@
2# Makefile for the s390 PCI subsystem. 2# Makefile for the s390 PCI subsystem.
3# 3#
4 4
5obj-$(CONFIG_PCI) += pci.o pci_dma.o pci_clp.o pci_msi.o pci_sysfs.o \ 5obj-$(CONFIG_PCI) += pci.o pci_dma.o pci_clp.o pci_sysfs.o \
6 pci_event.o pci_debug.o pci_insn.o 6 pci_event.o pci_debug.o pci_insn.o
diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
index e2956ad39a4f..f17a8343e360 100644
--- a/arch/s390/pci/pci.c
+++ b/arch/s390/pci/pci.c
@@ -42,45 +42,26 @@
42#define SIC_IRQ_MODE_SINGLE 1 42#define SIC_IRQ_MODE_SINGLE 1
43 43
44#define ZPCI_NR_DMA_SPACES 1 44#define ZPCI_NR_DMA_SPACES 1
45#define ZPCI_MSI_VEC_BITS 6
46#define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS 45#define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS
47 46
48/* list of all detected zpci devices */ 47/* list of all detected zpci devices */
49LIST_HEAD(zpci_list); 48static LIST_HEAD(zpci_list);
50EXPORT_SYMBOL_GPL(zpci_list); 49static DEFINE_SPINLOCK(zpci_list_lock);
51DEFINE_MUTEX(zpci_list_lock);
52EXPORT_SYMBOL_GPL(zpci_list_lock);
53 50
54static struct pci_hp_callback_ops *hotplug_ops; 51static void zpci_enable_irq(struct irq_data *data);
52static void zpci_disable_irq(struct irq_data *data);
55 53
56static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); 54static struct irq_chip zpci_irq_chip = {
57static DEFINE_SPINLOCK(zpci_domain_lock); 55 .name = "zPCI",
58 56 .irq_unmask = zpci_enable_irq,
59struct callback { 57 .irq_mask = zpci_disable_irq,
60 irq_handler_t handler;
61 void *data;
62}; 58};
63 59
64struct zdev_irq_map { 60static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
65 unsigned long aibv; /* AI bit vector */ 61static DEFINE_SPINLOCK(zpci_domain_lock);
66 int msi_vecs; /* consecutive MSI-vectors used */
67 int __unused;
68 struct callback cb[ZPCI_NR_MSI_VECS]; /* callback handler array */
69 spinlock_t lock; /* protect callbacks against de-reg */
70};
71
72struct intr_bucket {
73 /* amap of adapters, one bit per dev, corresponds to one irq nr */
74 unsigned long *alloc;
75 /* AI summary bit, global page for all devices */
76 unsigned long *aisb;
77 /* pointer to aibv and callback data in zdev */
78 struct zdev_irq_map *imap[ZPCI_NR_DEVICES];
79 /* protects the whole bucket struct */
80 spinlock_t lock;
81};
82 62
83static struct intr_bucket *bucket; 63static struct airq_iv *zpci_aisb_iv;
64static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES];
84 65
85/* Adapter interrupt definitions */ 66/* Adapter interrupt definitions */
86static void zpci_irq_handler(struct airq_struct *airq); 67static void zpci_irq_handler(struct airq_struct *airq);
@@ -96,27 +77,8 @@ static DECLARE_BITMAP(zpci_iomap, ZPCI_IOMAP_MAX_ENTRIES);
96struct zpci_iomap_entry *zpci_iomap_start; 77struct zpci_iomap_entry *zpci_iomap_start;
97EXPORT_SYMBOL_GPL(zpci_iomap_start); 78EXPORT_SYMBOL_GPL(zpci_iomap_start);
98 79
99/* highest irq summary bit */
100static int __read_mostly aisb_max;
101
102static struct kmem_cache *zdev_irq_cache;
103static struct kmem_cache *zdev_fmb_cache; 80static struct kmem_cache *zdev_fmb_cache;
104 81
105static inline int irq_to_msi_nr(unsigned int irq)
106{
107 return irq & ZPCI_MSI_MASK;
108}
109
110static inline int irq_to_dev_nr(unsigned int irq)
111{
112 return irq >> ZPCI_MSI_VEC_BITS;
113}
114
115static inline struct zdev_irq_map *get_imap(unsigned int irq)
116{
117 return bucket->imap[irq_to_dev_nr(irq)];
118}
119
120struct zpci_dev *get_zdev(struct pci_dev *pdev) 82struct zpci_dev *get_zdev(struct pci_dev *pdev)
121{ 83{
122 return (struct zpci_dev *) pdev->sysdata; 84 return (struct zpci_dev *) pdev->sysdata;
@@ -126,22 +88,17 @@ struct zpci_dev *get_zdev_by_fid(u32 fid)
126{ 88{
127 struct zpci_dev *tmp, *zdev = NULL; 89 struct zpci_dev *tmp, *zdev = NULL;
128 90
129 mutex_lock(&zpci_list_lock); 91 spin_lock(&zpci_list_lock);
130 list_for_each_entry(tmp, &zpci_list, entry) { 92 list_for_each_entry(tmp, &zpci_list, entry) {
131 if (tmp->fid == fid) { 93 if (tmp->fid == fid) {
132 zdev = tmp; 94 zdev = tmp;
133 break; 95 break;
134 } 96 }
135 } 97 }
136 mutex_unlock(&zpci_list_lock); 98 spin_unlock(&zpci_list_lock);
137 return zdev; 99 return zdev;
138} 100}
139 101
140bool zpci_fid_present(u32 fid)
141{
142 return (get_zdev_by_fid(fid) != NULL) ? true : false;
143}
144
145static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus) 102static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
146{ 103{
147 return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL; 104 return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
@@ -160,8 +117,7 @@ int pci_proc_domain(struct pci_bus *bus)
160EXPORT_SYMBOL_GPL(pci_proc_domain); 117EXPORT_SYMBOL_GPL(pci_proc_domain);
161 118
162/* Modify PCI: Register adapter interruptions */ 119/* Modify PCI: Register adapter interruptions */
163static int zpci_register_airq(struct zpci_dev *zdev, unsigned int aisb, 120static int zpci_set_airq(struct zpci_dev *zdev)
164 u64 aibv)
165{ 121{
166 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); 122 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT);
167 struct zpci_fib *fib; 123 struct zpci_fib *fib;
@@ -172,14 +128,14 @@ static int zpci_register_airq(struct zpci_dev *zdev, unsigned int aisb,
172 return -ENOMEM; 128 return -ENOMEM;
173 129
174 fib->isc = PCI_ISC; 130 fib->isc = PCI_ISC;
175 fib->noi = zdev->irq_map->msi_vecs;
176 fib->sum = 1; /* enable summary notifications */ 131 fib->sum = 1; /* enable summary notifications */
177 fib->aibv = aibv; 132 fib->noi = airq_iv_end(zdev->aibv);
178 fib->aibvo = 0; /* every function has its own page */ 133 fib->aibv = (unsigned long) zdev->aibv->vector;
179 fib->aisb = (u64) bucket->aisb + aisb / 8; 134 fib->aibvo = 0; /* each zdev has its own interrupt vector */
180 fib->aisbo = aisb & ZPCI_MSI_MASK; 135 fib->aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8;
136 fib->aisbo = zdev->aisb & 63;
181 137
182 rc = s390pci_mod_fc(req, fib); 138 rc = zpci_mod_fc(req, fib);
183 pr_debug("%s mpcifc returned noi: %d\n", __func__, fib->noi); 139 pr_debug("%s mpcifc returned noi: %d\n", __func__, fib->noi);
184 140
185 free_page((unsigned long) fib); 141 free_page((unsigned long) fib);
@@ -209,7 +165,7 @@ static int mod_pci(struct zpci_dev *zdev, int fn, u8 dmaas, struct mod_pci_args
209 fib->iota = args->iota; 165 fib->iota = args->iota;
210 fib->fmb_addr = args->fmb_addr; 166 fib->fmb_addr = args->fmb_addr;
211 167
212 rc = s390pci_mod_fc(req, fib); 168 rc = zpci_mod_fc(req, fib);
213 free_page((unsigned long) fib); 169 free_page((unsigned long) fib);
214 return rc; 170 return rc;
215} 171}
@@ -234,7 +190,7 @@ int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
234} 190}
235 191
236/* Modify PCI: Unregister adapter interruptions */ 192/* Modify PCI: Unregister adapter interruptions */
237static int zpci_unregister_airq(struct zpci_dev *zdev) 193static int zpci_clear_airq(struct zpci_dev *zdev)
238{ 194{
239 struct mod_pci_args args = { 0, 0, 0, 0 }; 195 struct mod_pci_args args = { 0, 0, 0, 0 };
240 196
@@ -283,7 +239,7 @@ static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
283 u64 data; 239 u64 data;
284 int rc; 240 int rc;
285 241
286 rc = s390pci_load(&data, req, offset); 242 rc = zpci_load(&data, req, offset);
287 if (!rc) { 243 if (!rc) {
288 data = data << ((8 - len) * 8); 244 data = data << ((8 - len) * 8);
289 data = le64_to_cpu(data); 245 data = le64_to_cpu(data);
@@ -301,25 +257,46 @@ static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
301 257
302 data = cpu_to_le64(data); 258 data = cpu_to_le64(data);
303 data = data >> ((8 - len) * 8); 259 data = data >> ((8 - len) * 8);
304 rc = s390pci_store(data, req, offset); 260 rc = zpci_store(data, req, offset);
305 return rc; 261 return rc;
306} 262}
307 263
308void enable_irq(unsigned int irq) 264static int zpci_msi_set_mask_bits(struct msi_desc *msi, u32 mask, u32 flag)
265{
266 int offset, pos;
267 u32 mask_bits;
268
269 if (msi->msi_attrib.is_msix) {
270 offset = msi->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
271 PCI_MSIX_ENTRY_VECTOR_CTRL;
272 msi->masked = readl(msi->mask_base + offset);
273 writel(flag, msi->mask_base + offset);
274 } else if (msi->msi_attrib.maskbit) {
275 pos = (long) msi->mask_base;
276 pci_read_config_dword(msi->dev, pos, &mask_bits);
277 mask_bits &= ~(mask);
278 mask_bits |= flag & mask;
279 pci_write_config_dword(msi->dev, pos, mask_bits);
280 } else
281 return 0;
282
283 msi->msi_attrib.maskbit = !!flag;
284 return 1;
285}
286
287static void zpci_enable_irq(struct irq_data *data)
309{ 288{
310 struct msi_desc *msi = irq_get_msi_desc(irq); 289 struct msi_desc *msi = irq_get_msi_desc(data->irq);
311 290
312 zpci_msi_set_mask_bits(msi, 1, 0); 291 zpci_msi_set_mask_bits(msi, 1, 0);
313} 292}
314EXPORT_SYMBOL_GPL(enable_irq);
315 293
316void disable_irq(unsigned int irq) 294static void zpci_disable_irq(struct irq_data *data)
317{ 295{
318 struct msi_desc *msi = irq_get_msi_desc(irq); 296 struct msi_desc *msi = irq_get_msi_desc(data->irq);
319 297
320 zpci_msi_set_mask_bits(msi, 1, 1); 298 zpci_msi_set_mask_bits(msi, 1, 1);
321} 299}
322EXPORT_SYMBOL_GPL(disable_irq);
323 300
324void pcibios_fixup_bus(struct pci_bus *bus) 301void pcibios_fixup_bus(struct pci_bus *bus)
325{ 302{
@@ -404,152 +381,147 @@ static struct pci_ops pci_root_ops = {
404 .write = pci_write, 381 .write = pci_write,
405}; 382};
406 383
407/* store the last handled bit to implement fair scheduling of devices */
408static DEFINE_PER_CPU(unsigned long, next_sbit);
409
410static void zpci_irq_handler(struct airq_struct *airq) 384static void zpci_irq_handler(struct airq_struct *airq)
411{ 385{
412 unsigned long sbit, mbit, last = 0, start = __get_cpu_var(next_sbit); 386 unsigned long si, ai;
413 int rescan = 0, max = aisb_max; 387 struct airq_iv *aibv;
414 struct zdev_irq_map *imap; 388 int irqs_on = 0;
415 389
416 inc_irq_stat(IRQIO_PCI); 390 inc_irq_stat(IRQIO_PCI);
417 sbit = start; 391 for (si = 0;;) {
418 392 /* Scan adapter summary indicator bit vector */
419scan: 393 si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv));
420 /* find summary_bit */ 394 if (si == -1UL) {
421 for_each_set_bit_left_cont(sbit, bucket->aisb, max) { 395 if (irqs_on++)
422 clear_bit(63 - (sbit & 63), bucket->aisb + (sbit >> 6)); 396 /* End of second scan with interrupts on. */
423 last = sbit; 397 break;
398 /* First scan complete, reenable interrupts. */
399 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
400 si = 0;
401 continue;
402 }
424 403
425 /* find vector bit */ 404 /* Scan the adapter interrupt vector for this device. */
426 imap = bucket->imap[sbit]; 405 aibv = zpci_aibv[si];
427 for_each_set_bit_left(mbit, &imap->aibv, imap->msi_vecs) { 406 for (ai = 0;;) {
407 ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv));
408 if (ai == -1UL)
409 break;
428 inc_irq_stat(IRQIO_MSI); 410 inc_irq_stat(IRQIO_MSI);
429 clear_bit(63 - mbit, &imap->aibv); 411 airq_iv_lock(aibv, ai);
430 412 generic_handle_irq(airq_iv_get_data(aibv, ai));
431 spin_lock(&imap->lock); 413 airq_iv_unlock(aibv, ai);
432 if (imap->cb[mbit].handler)
433 imap->cb[mbit].handler(mbit,
434 imap->cb[mbit].data);
435 spin_unlock(&imap->lock);
436 } 414 }
437 } 415 }
438
439 if (rescan)
440 goto out;
441
442 /* scan the skipped bits */
443 if (start > 0) {
444 sbit = 0;
445 max = start;
446 start = 0;
447 goto scan;
448 }
449
450 /* enable interrupts again */
451 set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
452
453 /* check again to not lose initiative */
454 rmb();
455 max = aisb_max;
456 sbit = find_first_bit_left(bucket->aisb, max);
457 if (sbit != max) {
458 rescan++;
459 goto scan;
460 }
461out:
462 /* store next device bit to scan */
463 __get_cpu_var(next_sbit) = (++last >= aisb_max) ? 0 : last;
464} 416}
465 417
466/* msi_vecs - number of requested interrupts, 0 place function to error state */ 418int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
467static int zpci_setup_msi(struct pci_dev *pdev, int msi_vecs)
468{ 419{
469 struct zpci_dev *zdev = get_zdev(pdev); 420 struct zpci_dev *zdev = get_zdev(pdev);
470 unsigned int aisb, msi_nr; 421 unsigned int hwirq, irq, msi_vecs;
422 unsigned long aisb;
471 struct msi_desc *msi; 423 struct msi_desc *msi;
424 struct msi_msg msg;
472 int rc; 425 int rc;
473 426
474 /* store the number of used MSI vectors */ 427 pr_debug("%s: requesting %d MSI-X interrupts...", __func__, nvec);
475 zdev->irq_map->msi_vecs = min(msi_vecs, ZPCI_NR_MSI_VECS); 428 if (type != PCI_CAP_ID_MSIX && type != PCI_CAP_ID_MSI)
476 429 return -EINVAL;
477 spin_lock(&bucket->lock); 430 msi_vecs = min(nvec, ZPCI_MSI_VEC_MAX);
478 aisb = find_first_zero_bit(bucket->alloc, PAGE_SIZE); 431 msi_vecs = min_t(unsigned int, msi_vecs, CONFIG_PCI_NR_MSI);
479 /* alloc map exhausted? */
480 if (aisb == PAGE_SIZE) {
481 spin_unlock(&bucket->lock);
482 return -EIO;
483 }
484 set_bit(aisb, bucket->alloc);
485 spin_unlock(&bucket->lock);
486 432
433 /* Allocate adapter summary indicator bit */
434 rc = -EIO;
435 aisb = airq_iv_alloc_bit(zpci_aisb_iv);
436 if (aisb == -1UL)
437 goto out;
487 zdev->aisb = aisb; 438 zdev->aisb = aisb;
488 if (aisb + 1 > aisb_max)
489 aisb_max = aisb + 1;
490 439
491 /* wire up IRQ shortcut pointer */ 440 /* Create adapter interrupt vector */
492 bucket->imap[zdev->aisb] = zdev->irq_map; 441 rc = -ENOMEM;
493 pr_debug("%s: imap[%u] linked to %p\n", __func__, zdev->aisb, zdev->irq_map); 442 zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK);
443 if (!zdev->aibv)
444 goto out_si;
494 445
495 /* TODO: irq number 0 wont be found if we return less than requested MSIs. 446 /* Wire up shortcut pointer */
496 * ignore it for now and fix in common code. 447 zpci_aibv[aisb] = zdev->aibv;
497 */
498 msi_nr = aisb << ZPCI_MSI_VEC_BITS;
499 448
449 /* Request MSI interrupts */
450 hwirq = 0;
500 list_for_each_entry(msi, &pdev->msi_list, list) { 451 list_for_each_entry(msi, &pdev->msi_list, list) {
501 rc = zpci_setup_msi_irq(zdev, msi, msi_nr, 452 rc = -EIO;
502 aisb << ZPCI_MSI_VEC_BITS); 453 irq = irq_alloc_desc(0); /* Alloc irq on node 0 */
454 if (irq == NO_IRQ)
455 goto out_msi;
456 rc = irq_set_msi_desc(irq, msi);
503 if (rc) 457 if (rc)
504 return rc; 458 goto out_msi;
505 msi_nr++; 459 irq_set_chip_and_handler(irq, &zpci_irq_chip,
460 handle_simple_irq);
461 msg.data = hwirq;
462 msg.address_lo = zdev->msi_addr & 0xffffffff;
463 msg.address_hi = zdev->msi_addr >> 32;
464 write_msi_msg(irq, &msg);
465 airq_iv_set_data(zdev->aibv, hwirq, irq);
466 hwirq++;
506 } 467 }
507 468
508 rc = zpci_register_airq(zdev, aisb, (u64) &zdev->irq_map->aibv); 469 /* Enable adapter interrupts */
509 if (rc) { 470 rc = zpci_set_airq(zdev);
510 clear_bit(aisb, bucket->alloc); 471 if (rc)
511 dev_err(&pdev->dev, "register MSI failed with: %d\n", rc); 472 goto out_msi;
512 return rc; 473
474 return (msi_vecs == nvec) ? 0 : msi_vecs;
475
476out_msi:
477 list_for_each_entry(msi, &pdev->msi_list, list) {
478 if (hwirq-- == 0)
479 break;
480 irq_set_msi_desc(msi->irq, NULL);
481 irq_free_desc(msi->irq);
482 msi->msg.address_lo = 0;
483 msi->msg.address_hi = 0;
484 msi->msg.data = 0;
485 msi->irq = 0;
513 } 486 }
514 return (zdev->irq_map->msi_vecs == msi_vecs) ? 487 zpci_aibv[aisb] = NULL;
515 0 : zdev->irq_map->msi_vecs; 488 airq_iv_release(zdev->aibv);
489out_si:
490 airq_iv_free_bit(zpci_aisb_iv, aisb);
491out:
492 dev_err(&pdev->dev, "register MSI failed with: %d\n", rc);
493 return rc;
516} 494}
517 495
518static void zpci_teardown_msi(struct pci_dev *pdev) 496void arch_teardown_msi_irqs(struct pci_dev *pdev)
519{ 497{
520 struct zpci_dev *zdev = get_zdev(pdev); 498 struct zpci_dev *zdev = get_zdev(pdev);
521 struct msi_desc *msi; 499 struct msi_desc *msi;
522 int aisb, rc; 500 int rc;
523 501
524 rc = zpci_unregister_airq(zdev); 502 pr_info("%s: on pdev: %p\n", __func__, pdev);
503
504 /* Disable adapter interrupts */
505 rc = zpci_clear_airq(zdev);
525 if (rc) { 506 if (rc) {
526 dev_err(&pdev->dev, "deregister MSI failed with: %d\n", rc); 507 dev_err(&pdev->dev, "deregister MSI failed with: %d\n", rc);
527 return; 508 return;
528 } 509 }
529 510
530 msi = list_first_entry(&pdev->msi_list, struct msi_desc, list); 511 /* Release MSI interrupts */
531 aisb = irq_to_dev_nr(msi->irq); 512 list_for_each_entry(msi, &pdev->msi_list, list) {
532 513 zpci_msi_set_mask_bits(msi, 1, 1);
533 list_for_each_entry(msi, &pdev->msi_list, list) 514 irq_set_msi_desc(msi->irq, NULL);
534 zpci_teardown_msi_irq(zdev, msi); 515 irq_free_desc(msi->irq);
535 516 msi->msg.address_lo = 0;
536 clear_bit(aisb, bucket->alloc); 517 msi->msg.address_hi = 0;
537 if (aisb + 1 == aisb_max) 518 msi->msg.data = 0;
538 aisb_max--; 519 msi->irq = 0;
539} 520 }
540
541int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
542{
543 pr_debug("%s: requesting %d MSI-X interrupts...", __func__, nvec);
544 if (type != PCI_CAP_ID_MSIX && type != PCI_CAP_ID_MSI)
545 return -EINVAL;
546 return zpci_setup_msi(pdev, nvec);
547}
548 521
549void arch_teardown_msi_irqs(struct pci_dev *pdev) 522 zpci_aibv[zdev->aisb] = NULL;
550{ 523 airq_iv_release(zdev->aibv);
551 pr_info("%s: on pdev: %p\n", __func__, pdev); 524 airq_iv_free_bit(zpci_aisb_iv, zdev->aisb);
552 zpci_teardown_msi(pdev);
553} 525}
554 526
555static void zpci_map_resources(struct zpci_dev *zdev) 527static void zpci_map_resources(struct zpci_dev *zdev)
@@ -564,8 +536,6 @@ static void zpci_map_resources(struct zpci_dev *zdev)
564 continue; 536 continue;
565 pdev->resource[i].start = (resource_size_t) pci_iomap(pdev, i, 0); 537 pdev->resource[i].start = (resource_size_t) pci_iomap(pdev, i, 0);
566 pdev->resource[i].end = pdev->resource[i].start + len - 1; 538 pdev->resource[i].end = pdev->resource[i].start + len - 1;
567 pr_debug("BAR%i: -> start: %Lx end: %Lx\n",
568 i, pdev->resource[i].start, pdev->resource[i].end);
569 } 539 }
570} 540}
571 541
@@ -589,162 +559,47 @@ struct zpci_dev *zpci_alloc_device(void)
589 559
590 /* Alloc memory for our private pci device data */ 560 /* Alloc memory for our private pci device data */
591 zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); 561 zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);
592 if (!zdev) 562 return zdev ? : ERR_PTR(-ENOMEM);
593 return ERR_PTR(-ENOMEM);
594
595 /* Alloc aibv & callback space */
596 zdev->irq_map = kmem_cache_zalloc(zdev_irq_cache, GFP_KERNEL);
597 if (!zdev->irq_map)
598 goto error;
599 WARN_ON((u64) zdev->irq_map & 0xff);
600 return zdev;
601
602error:
603 kfree(zdev);
604 return ERR_PTR(-ENOMEM);
605} 563}
606 564
607void zpci_free_device(struct zpci_dev *zdev) 565void zpci_free_device(struct zpci_dev *zdev)
608{ 566{
609 kmem_cache_free(zdev_irq_cache, zdev->irq_map);
610 kfree(zdev); 567 kfree(zdev);
611} 568}
612 569
613/*
614 * Too late for any s390 specific setup, since interrupts must be set up
615 * already which requires DMA setup too and the pci scan will access the
616 * config space, which only works if the function handle is enabled.
617 */
618int pcibios_enable_device(struct pci_dev *pdev, int mask)
619{
620 struct resource *res;
621 u16 cmd;
622 int i;
623
624 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
625
626 for (i = 0; i < PCI_BAR_COUNT; i++) {
627 res = &pdev->resource[i];
628
629 if (res->flags & IORESOURCE_IO)
630 return -EINVAL;
631
632 if (res->flags & IORESOURCE_MEM)
633 cmd |= PCI_COMMAND_MEMORY;
634 }
635 pci_write_config_word(pdev, PCI_COMMAND, cmd);
636 return 0;
637}
638
639int pcibios_add_platform_entries(struct pci_dev *pdev) 570int pcibios_add_platform_entries(struct pci_dev *pdev)
640{ 571{
641 return zpci_sysfs_add_device(&pdev->dev); 572 return zpci_sysfs_add_device(&pdev->dev);
642} 573}
643 574
644int zpci_request_irq(unsigned int irq, irq_handler_t handler, void *data)
645{
646 int msi_nr = irq_to_msi_nr(irq);
647 struct zdev_irq_map *imap;
648 struct msi_desc *msi;
649
650 msi = irq_get_msi_desc(irq);
651 if (!msi)
652 return -EIO;
653
654 imap = get_imap(irq);
655 spin_lock_init(&imap->lock);
656
657 pr_debug("%s: register handler for IRQ:MSI %d:%d\n", __func__, irq >> 6, msi_nr);
658 imap->cb[msi_nr].handler = handler;
659 imap->cb[msi_nr].data = data;
660
661 /*
662 * The generic MSI code returns with the interrupt disabled on the
663 * card, using the MSI mask bits. Firmware doesn't appear to unmask
664 * at that level, so we do it here by hand.
665 */
666 zpci_msi_set_mask_bits(msi, 1, 0);
667 return 0;
668}
669
670void zpci_free_irq(unsigned int irq)
671{
672 struct zdev_irq_map *imap = get_imap(irq);
673 int msi_nr = irq_to_msi_nr(irq);
674 unsigned long flags;
675
676 pr_debug("%s: for irq: %d\n", __func__, irq);
677
678 spin_lock_irqsave(&imap->lock, flags);
679 imap->cb[msi_nr].handler = NULL;
680 imap->cb[msi_nr].data = NULL;
681 spin_unlock_irqrestore(&imap->lock, flags);
682}
683
684int request_irq(unsigned int irq, irq_handler_t handler,
685 unsigned long irqflags, const char *devname, void *dev_id)
686{
687 pr_debug("%s: irq: %d handler: %p flags: %lx dev: %s\n",
688 __func__, irq, handler, irqflags, devname);
689
690 return zpci_request_irq(irq, handler, dev_id);
691}
692EXPORT_SYMBOL_GPL(request_irq);
693
694void free_irq(unsigned int irq, void *dev_id)
695{
696 zpci_free_irq(irq);
697}
698EXPORT_SYMBOL_GPL(free_irq);
699
700static int __init zpci_irq_init(void) 575static int __init zpci_irq_init(void)
701{ 576{
702 int cpu, rc; 577 int rc;
703
704 bucket = kzalloc(sizeof(*bucket), GFP_KERNEL);
705 if (!bucket)
706 return -ENOMEM;
707
708 bucket->aisb = (unsigned long *) get_zeroed_page(GFP_KERNEL);
709 if (!bucket->aisb) {
710 rc = -ENOMEM;
711 goto out_aisb;
712 }
713
714 bucket->alloc = (unsigned long *) get_zeroed_page(GFP_KERNEL);
715 if (!bucket->alloc) {
716 rc = -ENOMEM;
717 goto out_alloc;
718 }
719 578
720 rc = register_adapter_interrupt(&zpci_airq); 579 rc = register_adapter_interrupt(&zpci_airq);
721 if (rc) 580 if (rc)
722 goto out_ai; 581 goto out;
723 /* Set summary to 1 to be called every time for the ISC. */ 582 /* Set summary to 1 to be called every time for the ISC. */
724 *zpci_airq.lsi_ptr = 1; 583 *zpci_airq.lsi_ptr = 1;
725 584
726 for_each_online_cpu(cpu) 585 rc = -ENOMEM;
727 per_cpu(next_sbit, cpu) = 0; 586 zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC);
587 if (!zpci_aisb_iv)
588 goto out_airq;
728 589
729 spin_lock_init(&bucket->lock); 590 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
730 set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC);
731 return 0; 591 return 0;
732 592
733out_ai: 593out_airq:
734 free_page((unsigned long) bucket->alloc); 594 unregister_adapter_interrupt(&zpci_airq);
735out_alloc: 595out:
736 free_page((unsigned long) bucket->aisb);
737out_aisb:
738 kfree(bucket);
739 return rc; 596 return rc;
740} 597}
741 598
742static void zpci_irq_exit(void) 599static void zpci_irq_exit(void)
743{ 600{
744 free_page((unsigned long) bucket->alloc); 601 airq_iv_release(zpci_aisb_iv);
745 free_page((unsigned long) bucket->aisb);
746 unregister_adapter_interrupt(&zpci_airq); 602 unregister_adapter_interrupt(&zpci_airq);
747 kfree(bucket);
748} 603}
749 604
750static struct resource *zpci_alloc_bus_resource(unsigned long start, unsigned long size, 605static struct resource *zpci_alloc_bus_resource(unsigned long start, unsigned long size,
@@ -801,16 +656,49 @@ static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
801int pcibios_add_device(struct pci_dev *pdev) 656int pcibios_add_device(struct pci_dev *pdev)
802{ 657{
803 struct zpci_dev *zdev = get_zdev(pdev); 658 struct zpci_dev *zdev = get_zdev(pdev);
659 struct resource *res;
660 int i;
661
662 zdev->pdev = pdev;
663 zpci_map_resources(zdev);
664
665 for (i = 0; i < PCI_BAR_COUNT; i++) {
666 res = &pdev->resource[i];
667 if (res->parent || !res->flags)
668 continue;
669 pci_claim_resource(pdev, i);
670 }
671
672 return 0;
673}
674
675int pcibios_enable_device(struct pci_dev *pdev, int mask)
676{
677 struct zpci_dev *zdev = get_zdev(pdev);
678 struct resource *res;
679 u16 cmd;
680 int i;
804 681
805 zdev->pdev = pdev; 682 zdev->pdev = pdev;
806 zpci_debug_init_device(zdev); 683 zpci_debug_init_device(zdev);
807 zpci_fmb_enable_device(zdev); 684 zpci_fmb_enable_device(zdev);
808 zpci_map_resources(zdev); 685 zpci_map_resources(zdev);
809 686
687 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
688 for (i = 0; i < PCI_BAR_COUNT; i++) {
689 res = &pdev->resource[i];
690
691 if (res->flags & IORESOURCE_IO)
692 return -EINVAL;
693
694 if (res->flags & IORESOURCE_MEM)
695 cmd |= PCI_COMMAND_MEMORY;
696 }
697 pci_write_config_word(pdev, PCI_COMMAND, cmd);
810 return 0; 698 return 0;
811} 699}
812 700
813void pcibios_release_device(struct pci_dev *pdev) 701void pcibios_disable_device(struct pci_dev *pdev)
814{ 702{
815 struct zpci_dev *zdev = get_zdev(pdev); 703 struct zpci_dev *zdev = get_zdev(pdev);
816 704
@@ -898,6 +786,8 @@ int zpci_enable_device(struct zpci_dev *zdev)
898 rc = zpci_dma_init_device(zdev); 786 rc = zpci_dma_init_device(zdev);
899 if (rc) 787 if (rc)
900 goto out_dma; 788 goto out_dma;
789
790 zdev->state = ZPCI_FN_STATE_ONLINE;
901 return 0; 791 return 0;
902 792
903out_dma: 793out_dma:
@@ -926,18 +816,16 @@ int zpci_create_device(struct zpci_dev *zdev)
926 rc = zpci_enable_device(zdev); 816 rc = zpci_enable_device(zdev);
927 if (rc) 817 if (rc)
928 goto out_free; 818 goto out_free;
929
930 zdev->state = ZPCI_FN_STATE_ONLINE;
931 } 819 }
932 rc = zpci_scan_bus(zdev); 820 rc = zpci_scan_bus(zdev);
933 if (rc) 821 if (rc)
934 goto out_disable; 822 goto out_disable;
935 823
936 mutex_lock(&zpci_list_lock); 824 spin_lock(&zpci_list_lock);
937 list_add_tail(&zdev->entry, &zpci_list); 825 list_add_tail(&zdev->entry, &zpci_list);
938 if (hotplug_ops) 826 spin_unlock(&zpci_list_lock);
939 hotplug_ops->create_slot(zdev); 827
940 mutex_unlock(&zpci_list_lock); 828 zpci_init_slot(zdev);
941 829
942 return 0; 830 return 0;
943 831
@@ -967,15 +855,10 @@ static inline int barsize(u8 size)
967 855
968static int zpci_mem_init(void) 856static int zpci_mem_init(void)
969{ 857{
970 zdev_irq_cache = kmem_cache_create("PCI_IRQ_cache", sizeof(struct zdev_irq_map),
971 L1_CACHE_BYTES, SLAB_HWCACHE_ALIGN, NULL);
972 if (!zdev_irq_cache)
973 goto error_zdev;
974
975 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 858 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
976 16, 0, NULL); 859 16, 0, NULL);
977 if (!zdev_fmb_cache) 860 if (!zdev_fmb_cache)
978 goto error_fmb; 861 goto error_zdev;
979 862
980 /* TODO: use realloc */ 863 /* TODO: use realloc */
981 zpci_iomap_start = kzalloc(ZPCI_IOMAP_MAX_ENTRIES * sizeof(*zpci_iomap_start), 864 zpci_iomap_start = kzalloc(ZPCI_IOMAP_MAX_ENTRIES * sizeof(*zpci_iomap_start),
@@ -986,8 +869,6 @@ static int zpci_mem_init(void)
986 869
987error_iomap: 870error_iomap:
988 kmem_cache_destroy(zdev_fmb_cache); 871 kmem_cache_destroy(zdev_fmb_cache);
989error_fmb:
990 kmem_cache_destroy(zdev_irq_cache);
991error_zdev: 872error_zdev:
992 return -ENOMEM; 873 return -ENOMEM;
993} 874}
@@ -995,28 +876,10 @@ error_zdev:
995static void zpci_mem_exit(void) 876static void zpci_mem_exit(void)
996{ 877{
997 kfree(zpci_iomap_start); 878 kfree(zpci_iomap_start);
998 kmem_cache_destroy(zdev_irq_cache);
999 kmem_cache_destroy(zdev_fmb_cache); 879 kmem_cache_destroy(zdev_fmb_cache);
1000} 880}
1001 881
1002void zpci_register_hp_ops(struct pci_hp_callback_ops *ops) 882static unsigned int s390_pci_probe;
1003{
1004 mutex_lock(&zpci_list_lock);
1005 hotplug_ops = ops;
1006 mutex_unlock(&zpci_list_lock);
1007}
1008EXPORT_SYMBOL_GPL(zpci_register_hp_ops);
1009
1010void zpci_deregister_hp_ops(void)
1011{
1012 mutex_lock(&zpci_list_lock);
1013 hotplug_ops = NULL;
1014 mutex_unlock(&zpci_list_lock);
1015}
1016EXPORT_SYMBOL_GPL(zpci_deregister_hp_ops);
1017
1018unsigned int s390_pci_probe;
1019EXPORT_SYMBOL_GPL(s390_pci_probe);
1020 883
1021char * __init pcibios_setup(char *str) 884char * __init pcibios_setup(char *str)
1022{ 885{
@@ -1044,16 +907,12 @@ static int __init pci_base_init(void)
1044 907
1045 rc = zpci_debug_init(); 908 rc = zpci_debug_init();
1046 if (rc) 909 if (rc)
1047 return rc; 910 goto out;
1048 911
1049 rc = zpci_mem_init(); 912 rc = zpci_mem_init();
1050 if (rc) 913 if (rc)
1051 goto out_mem; 914 goto out_mem;
1052 915
1053 rc = zpci_msihash_init();
1054 if (rc)
1055 goto out_hash;
1056
1057 rc = zpci_irq_init(); 916 rc = zpci_irq_init();
1058 if (rc) 917 if (rc)
1059 goto out_irq; 918 goto out_irq;
@@ -1062,7 +921,7 @@ static int __init pci_base_init(void)
1062 if (rc) 921 if (rc)
1063 goto out_dma; 922 goto out_dma;
1064 923
1065 rc = clp_find_pci_devices(); 924 rc = clp_scan_pci_devices();
1066 if (rc) 925 if (rc)
1067 goto out_find; 926 goto out_find;
1068 927
@@ -1073,11 +932,15 @@ out_find:
1073out_dma: 932out_dma:
1074 zpci_irq_exit(); 933 zpci_irq_exit();
1075out_irq: 934out_irq:
1076 zpci_msihash_exit();
1077out_hash:
1078 zpci_mem_exit(); 935 zpci_mem_exit();
1079out_mem: 936out_mem:
1080 zpci_debug_exit(); 937 zpci_debug_exit();
938out:
1081 return rc; 939 return rc;
1082} 940}
1083subsys_initcall(pci_base_init); 941subsys_initcall_sync(pci_base_init);
942
943void zpci_rescan(void)
944{
945 clp_rescan_pci_devices_simple();
946}
diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c
index 2e9539625d93..475563c3d1e4 100644
--- a/arch/s390/pci/pci_clp.c
+++ b/arch/s390/pci/pci_clp.c
@@ -36,9 +36,9 @@ static inline u8 clp_instr(void *data)
36 return cc; 36 return cc;
37} 37}
38 38
39static void *clp_alloc_block(void) 39static void *clp_alloc_block(gfp_t gfp_mask)
40{ 40{
41 return (void *) __get_free_pages(GFP_KERNEL, get_order(CLP_BLK_SIZE)); 41 return (void *) __get_free_pages(gfp_mask, get_order(CLP_BLK_SIZE));
42} 42}
43 43
44static void clp_free_block(void *ptr) 44static void clp_free_block(void *ptr)
@@ -70,7 +70,7 @@ static int clp_query_pci_fngrp(struct zpci_dev *zdev, u8 pfgid)
70 struct clp_req_rsp_query_pci_grp *rrb; 70 struct clp_req_rsp_query_pci_grp *rrb;
71 int rc; 71 int rc;
72 72
73 rrb = clp_alloc_block(); 73 rrb = clp_alloc_block(GFP_KERNEL);
74 if (!rrb) 74 if (!rrb)
75 return -ENOMEM; 75 return -ENOMEM;
76 76
@@ -113,7 +113,7 @@ static int clp_query_pci_fn(struct zpci_dev *zdev, u32 fh)
113 struct clp_req_rsp_query_pci *rrb; 113 struct clp_req_rsp_query_pci *rrb;
114 int rc; 114 int rc;
115 115
116 rrb = clp_alloc_block(); 116 rrb = clp_alloc_block(GFP_KERNEL);
117 if (!rrb) 117 if (!rrb)
118 return -ENOMEM; 118 return -ENOMEM;
119 119
@@ -179,9 +179,9 @@ error:
179static int clp_set_pci_fn(u32 *fh, u8 nr_dma_as, u8 command) 179static int clp_set_pci_fn(u32 *fh, u8 nr_dma_as, u8 command)
180{ 180{
181 struct clp_req_rsp_set_pci *rrb; 181 struct clp_req_rsp_set_pci *rrb;
182 int rc, retries = 1000; 182 int rc, retries = 100;
183 183
184 rrb = clp_alloc_block(); 184 rrb = clp_alloc_block(GFP_KERNEL);
185 if (!rrb) 185 if (!rrb)
186 return -ENOMEM; 186 return -ENOMEM;
187 187
@@ -199,7 +199,7 @@ static int clp_set_pci_fn(u32 *fh, u8 nr_dma_as, u8 command)
199 retries--; 199 retries--;
200 if (retries < 0) 200 if (retries < 0)
201 break; 201 break;
202 msleep(1); 202 msleep(20);
203 } 203 }
204 } while (rrb->response.hdr.rsp == CLP_RC_SETPCIFN_BUSY); 204 } while (rrb->response.hdr.rsp == CLP_RC_SETPCIFN_BUSY);
205 205
@@ -245,49 +245,12 @@ int clp_disable_fh(struct zpci_dev *zdev)
245 return rc; 245 return rc;
246} 246}
247 247
248static void clp_check_pcifn_entry(struct clp_fh_list_entry *entry) 248static int clp_list_pci(struct clp_req_rsp_list_pci *rrb,
249 void (*cb)(struct clp_fh_list_entry *entry))
249{ 250{
250 int present, rc;
251
252 if (!entry->vendor_id)
253 return;
254
255 /* TODO: be a little bit more scalable */
256 present = zpci_fid_present(entry->fid);
257
258 if (present)
259 pr_debug("%s: device %x already present\n", __func__, entry->fid);
260
261 /* skip already used functions */
262 if (present && entry->config_state)
263 return;
264
265 /* aev 306: function moved to stand-by state */
266 if (present && !entry->config_state) {
267 /*
268 * The handle is already disabled, that means no iota/irq freeing via
269 * the firmware interfaces anymore. Need to free resources manually
270 * (DMA memory, debug, sysfs)...
271 */
272 zpci_stop_device(get_zdev_by_fid(entry->fid));
273 return;
274 }
275
276 rc = clp_add_pci_device(entry->fid, entry->fh, entry->config_state);
277 if (rc)
278 pr_err("Failed to add fid: 0x%x\n", entry->fid);
279}
280
281int clp_find_pci_devices(void)
282{
283 struct clp_req_rsp_list_pci *rrb;
284 u64 resume_token = 0; 251 u64 resume_token = 0;
285 int entries, i, rc; 252 int entries, i, rc;
286 253
287 rrb = clp_alloc_block();
288 if (!rrb)
289 return -ENOMEM;
290
291 do { 254 do {
292 memset(rrb, 0, sizeof(*rrb)); 255 memset(rrb, 0, sizeof(*rrb));
293 rrb->request.hdr.len = sizeof(rrb->request); 256 rrb->request.hdr.len = sizeof(rrb->request);
@@ -316,12 +279,101 @@ int clp_find_pci_devices(void)
316 resume_token = rrb->response.resume_token; 279 resume_token = rrb->response.resume_token;
317 280
318 for (i = 0; i < entries; i++) 281 for (i = 0; i < entries; i++)
319 clp_check_pcifn_entry(&rrb->response.fh_list[i]); 282 cb(&rrb->response.fh_list[i]);
320 } while (resume_token); 283 } while (resume_token);
321 284
322 pr_debug("Maximum number of supported PCI functions: %u\n", 285 pr_debug("Maximum number of supported PCI functions: %u\n",
323 rrb->response.max_fn); 286 rrb->response.max_fn);
324out: 287out:
288 return rc;
289}
290
291static void __clp_add(struct clp_fh_list_entry *entry)
292{
293 if (!entry->vendor_id)
294 return;
295
296 clp_add_pci_device(entry->fid, entry->fh, entry->config_state);
297}
298
299static void __clp_rescan(struct clp_fh_list_entry *entry)
300{
301 struct zpci_dev *zdev;
302
303 if (!entry->vendor_id)
304 return;
305
306 zdev = get_zdev_by_fid(entry->fid);
307 if (!zdev) {
308 clp_add_pci_device(entry->fid, entry->fh, entry->config_state);
309 return;
310 }
311
312 if (!entry->config_state) {
313 /*
314 * The handle is already disabled, that means no iota/irq freeing via
315 * the firmware interfaces anymore. Need to free resources manually
316 * (DMA memory, debug, sysfs)...
317 */
318 zpci_stop_device(zdev);
319 }
320}
321
322static void __clp_update(struct clp_fh_list_entry *entry)
323{
324 struct zpci_dev *zdev;
325
326 if (!entry->vendor_id)
327 return;
328
329 zdev = get_zdev_by_fid(entry->fid);
330 if (!zdev)
331 return;
332
333 zdev->fh = entry->fh;
334}
335
336int clp_scan_pci_devices(void)
337{
338 struct clp_req_rsp_list_pci *rrb;
339 int rc;
340
341 rrb = clp_alloc_block(GFP_KERNEL);
342 if (!rrb)
343 return -ENOMEM;
344
345 rc = clp_list_pci(rrb, __clp_add);
346
347 clp_free_block(rrb);
348 return rc;
349}
350
351int clp_rescan_pci_devices(void)
352{
353 struct clp_req_rsp_list_pci *rrb;
354 int rc;
355
356 rrb = clp_alloc_block(GFP_KERNEL);
357 if (!rrb)
358 return -ENOMEM;
359
360 rc = clp_list_pci(rrb, __clp_rescan);
361
362 clp_free_block(rrb);
363 return rc;
364}
365
366int clp_rescan_pci_devices_simple(void)
367{
368 struct clp_req_rsp_list_pci *rrb;
369 int rc;
370
371 rrb = clp_alloc_block(GFP_NOWAIT);
372 if (!rrb)
373 return -ENOMEM;
374
375 rc = clp_list_pci(rrb, __clp_update);
376
325 clp_free_block(rrb); 377 clp_free_block(rrb);
326 return rc; 378 return rc;
327} 379}
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index a2343c1f6e04..7e5573acb063 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -10,6 +10,7 @@
10#include <linux/export.h> 10#include <linux/export.h>
11#include <linux/iommu-helper.h> 11#include <linux/iommu-helper.h>
12#include <linux/dma-mapping.h> 12#include <linux/dma-mapping.h>
13#include <linux/vmalloc.h>
13#include <linux/pci.h> 14#include <linux/pci.h>
14#include <asm/pci_dma.h> 15#include <asm/pci_dma.h>
15 16
@@ -170,8 +171,8 @@ static int dma_update_trans(struct zpci_dev *zdev, unsigned long pa,
170 */ 171 */
171 goto no_refresh; 172 goto no_refresh;
172 173
173 rc = s390pci_refresh_trans((u64) zdev->fh << 32, start_dma_addr, 174 rc = zpci_refresh_trans((u64) zdev->fh << 32, start_dma_addr,
174 nr_pages * PAGE_SIZE); 175 nr_pages * PAGE_SIZE);
175 176
176no_refresh: 177no_refresh:
177 spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags); 178 spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags);
@@ -407,7 +408,6 @@ static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
407 408
408int zpci_dma_init_device(struct zpci_dev *zdev) 409int zpci_dma_init_device(struct zpci_dev *zdev)
409{ 410{
410 unsigned int bitmap_order;
411 int rc; 411 int rc;
412 412
413 spin_lock_init(&zdev->iommu_bitmap_lock); 413 spin_lock_init(&zdev->iommu_bitmap_lock);
@@ -421,12 +421,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev)
421 421
422 zdev->iommu_size = (unsigned long) high_memory - PAGE_OFFSET; 422 zdev->iommu_size = (unsigned long) high_memory - PAGE_OFFSET;
423 zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT; 423 zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT;
424 bitmap_order = get_order(zdev->iommu_pages / 8); 424 zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8);
425 pr_info("iommu_size: 0x%lx iommu_pages: 0x%lx bitmap_order: %i\n",
426 zdev->iommu_size, zdev->iommu_pages, bitmap_order);
427
428 zdev->iommu_bitmap = (void *) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
429 bitmap_order);
430 if (!zdev->iommu_bitmap) { 425 if (!zdev->iommu_bitmap) {
431 rc = -ENOMEM; 426 rc = -ENOMEM;
432 goto out_reg; 427 goto out_reg;
@@ -451,8 +446,7 @@ void zpci_dma_exit_device(struct zpci_dev *zdev)
451{ 446{
452 zpci_unregister_ioat(zdev, 0); 447 zpci_unregister_ioat(zdev, 0);
453 dma_cleanup_tables(zdev); 448 dma_cleanup_tables(zdev);
454 free_pages((unsigned long) zdev->iommu_bitmap, 449 vfree(zdev->iommu_bitmap);
455 get_order(zdev->iommu_pages / 8));
456 zdev->iommu_bitmap = NULL; 450 zdev->iommu_bitmap = NULL;
457 zdev->next_bit = 0; 451 zdev->next_bit = 0;
458} 452}
diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c
index ec62e3a0dc09..0aecaf954845 100644
--- a/arch/s390/pci/pci_event.c
+++ b/arch/s390/pci/pci_event.c
@@ -69,7 +69,7 @@ static void zpci_event_log_avail(struct zpci_ccdf_avail *ccdf)
69 clp_add_pci_device(ccdf->fid, ccdf->fh, 0); 69 clp_add_pci_device(ccdf->fid, ccdf->fh, 0);
70 break; 70 break;
71 case 0x0306: 71 case 0x0306:
72 clp_find_pci_devices(); 72 clp_rescan_pci_devices();
73 break; 73 break;
74 default: 74 default:
75 break; 75 break;
diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c
index 22eeb9d7ffeb..85267c058af8 100644
--- a/arch/s390/pci/pci_insn.c
+++ b/arch/s390/pci/pci_insn.c
@@ -27,7 +27,7 @@ static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status)
27 return cc; 27 return cc;
28} 28}
29 29
30int s390pci_mod_fc(u64 req, struct zpci_fib *fib) 30int zpci_mod_fc(u64 req, struct zpci_fib *fib)
31{ 31{
32 u8 cc, status; 32 u8 cc, status;
33 33
@@ -61,7 +61,7 @@ static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status)
61 return cc; 61 return cc;
62} 62}
63 63
64int s390pci_refresh_trans(u64 fn, u64 addr, u64 range) 64int zpci_refresh_trans(u64 fn, u64 addr, u64 range)
65{ 65{
66 u8 cc, status; 66 u8 cc, status;
67 67
@@ -78,7 +78,7 @@ int s390pci_refresh_trans(u64 fn, u64 addr, u64 range)
78} 78}
79 79
80/* Set Interruption Controls */ 80/* Set Interruption Controls */
81void set_irq_ctrl(u16 ctl, char *unused, u8 isc) 81void zpci_set_irq_ctrl(u16 ctl, char *unused, u8 isc)
82{ 82{
83 asm volatile ( 83 asm volatile (
84 " .insn rsy,0xeb00000000d1,%[ctl],%[isc],%[u]\n" 84 " .insn rsy,0xeb00000000d1,%[ctl],%[isc],%[u]\n"
@@ -109,7 +109,7 @@ static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status)
109 return cc; 109 return cc;
110} 110}
111 111
112int s390pci_load(u64 *data, u64 req, u64 offset) 112int zpci_load(u64 *data, u64 req, u64 offset)
113{ 113{
114 u8 status; 114 u8 status;
115 int cc; 115 int cc;
@@ -125,7 +125,7 @@ int s390pci_load(u64 *data, u64 req, u64 offset)
125 __func__, cc, status, req, offset); 125 __func__, cc, status, req, offset);
126 return (cc > 0) ? -EIO : cc; 126 return (cc > 0) ? -EIO : cc;
127} 127}
128EXPORT_SYMBOL_GPL(s390pci_load); 128EXPORT_SYMBOL_GPL(zpci_load);
129 129
130/* PCI Store */ 130/* PCI Store */
131static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) 131static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status)
@@ -147,7 +147,7 @@ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status)
147 return cc; 147 return cc;
148} 148}
149 149
150int s390pci_store(u64 data, u64 req, u64 offset) 150int zpci_store(u64 data, u64 req, u64 offset)
151{ 151{
152 u8 status; 152 u8 status;
153 int cc; 153 int cc;
@@ -163,7 +163,7 @@ int s390pci_store(u64 data, u64 req, u64 offset)
163 __func__, cc, status, req, offset); 163 __func__, cc, status, req, offset);
164 return (cc > 0) ? -EIO : cc; 164 return (cc > 0) ? -EIO : cc;
165} 165}
166EXPORT_SYMBOL_GPL(s390pci_store); 166EXPORT_SYMBOL_GPL(zpci_store);
167 167
168/* PCI Store Block */ 168/* PCI Store Block */
169static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) 169static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status)
@@ -183,7 +183,7 @@ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status)
183 return cc; 183 return cc;
184} 184}
185 185
186int s390pci_store_block(const u64 *data, u64 req, u64 offset) 186int zpci_store_block(const u64 *data, u64 req, u64 offset)
187{ 187{
188 u8 status; 188 u8 status;
189 int cc; 189 int cc;
@@ -199,4 +199,4 @@ int s390pci_store_block(const u64 *data, u64 req, u64 offset)
199 __func__, cc, status, req, offset); 199 __func__, cc, status, req, offset);
200 return (cc > 0) ? -EIO : cc; 200 return (cc > 0) ? -EIO : cc;
201} 201}
202EXPORT_SYMBOL_GPL(s390pci_store_block); 202EXPORT_SYMBOL_GPL(zpci_store_block);
diff --git a/arch/s390/pci/pci_msi.c b/arch/s390/pci/pci_msi.c
deleted file mode 100644
index b097aed05a9b..000000000000
--- a/arch/s390/pci/pci_msi.c
+++ /dev/null
@@ -1,142 +0,0 @@
1/*
2 * Copyright IBM Corp. 2012
3 *
4 * Author(s):
5 * Jan Glauber <jang@linux.vnet.ibm.com>
6 */
7
8#define COMPONENT "zPCI"
9#define pr_fmt(fmt) COMPONENT ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/err.h>
13#include <linux/rculist.h>
14#include <linux/hash.h>
15#include <linux/pci.h>
16#include <linux/msi.h>
17#include <asm/hw_irq.h>
18
19/* mapping of irq numbers to msi_desc */
20static struct hlist_head *msi_hash;
21static const unsigned int msi_hash_bits = 8;
22#define MSI_HASH_BUCKETS (1U << msi_hash_bits)
23#define msi_hashfn(nr) hash_long(nr, msi_hash_bits)
24
25static DEFINE_SPINLOCK(msi_map_lock);
26
27struct msi_desc *__irq_get_msi_desc(unsigned int irq)
28{
29 struct msi_map *map;
30
31 hlist_for_each_entry_rcu(map,
32 &msi_hash[msi_hashfn(irq)], msi_chain)
33 if (map->irq == irq)
34 return map->msi;
35 return NULL;
36}
37
38int zpci_msi_set_mask_bits(struct msi_desc *msi, u32 mask, u32 flag)
39{
40 if (msi->msi_attrib.is_msix) {
41 int offset = msi->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
42 PCI_MSIX_ENTRY_VECTOR_CTRL;
43 msi->masked = readl(msi->mask_base + offset);
44 writel(flag, msi->mask_base + offset);
45 } else {
46 if (msi->msi_attrib.maskbit) {
47 int pos;
48 u32 mask_bits;
49
50 pos = (long) msi->mask_base;
51 pci_read_config_dword(msi->dev, pos, &mask_bits);
52 mask_bits &= ~(mask);
53 mask_bits |= flag & mask;
54 pci_write_config_dword(msi->dev, pos, mask_bits);
55 } else {
56 return 0;
57 }
58 }
59
60 msi->msi_attrib.maskbit = !!flag;
61 return 1;
62}
63
64int zpci_setup_msi_irq(struct zpci_dev *zdev, struct msi_desc *msi,
65 unsigned int nr, int offset)
66{
67 struct msi_map *map;
68 struct msi_msg msg;
69 int rc;
70
71 map = kmalloc(sizeof(*map), GFP_KERNEL);
72 if (map == NULL)
73 return -ENOMEM;
74
75 map->irq = nr;
76 map->msi = msi;
77 zdev->msi_map[nr & ZPCI_MSI_MASK] = map;
78 INIT_HLIST_NODE(&map->msi_chain);
79
80 pr_debug("%s hashing irq: %u to bucket nr: %llu\n",
81 __func__, nr, msi_hashfn(nr));
82 hlist_add_head_rcu(&map->msi_chain, &msi_hash[msi_hashfn(nr)]);
83
84 spin_lock(&msi_map_lock);
85 rc = irq_set_msi_desc(nr, msi);
86 if (rc) {
87 spin_unlock(&msi_map_lock);
88 hlist_del_rcu(&map->msi_chain);
89 kfree(map);
90 zdev->msi_map[nr & ZPCI_MSI_MASK] = NULL;
91 return rc;
92 }
93 spin_unlock(&msi_map_lock);
94
95 msg.data = nr - offset;
96 msg.address_lo = zdev->msi_addr & 0xffffffff;
97 msg.address_hi = zdev->msi_addr >> 32;
98 write_msi_msg(nr, &msg);
99 return 0;
100}
101
102void zpci_teardown_msi_irq(struct zpci_dev *zdev, struct msi_desc *msi)
103{
104 int irq = msi->irq & ZPCI_MSI_MASK;
105 struct msi_map *map;
106
107 msi->msg.address_lo = 0;
108 msi->msg.address_hi = 0;
109 msi->msg.data = 0;
110 msi->irq = 0;
111 zpci_msi_set_mask_bits(msi, 1, 1);
112
113 spin_lock(&msi_map_lock);
114 map = zdev->msi_map[irq];
115 hlist_del_rcu(&map->msi_chain);
116 kfree(map);
117 zdev->msi_map[irq] = NULL;
118 spin_unlock(&msi_map_lock);
119}
120
121/*
122 * The msi hash table has 256 entries which is good for 4..20
123 * devices (a typical device allocates 10 + CPUs MSI's). Maybe make
124 * the hash table size adjustable later.
125 */
126int __init zpci_msihash_init(void)
127{
128 unsigned int i;
129
130 msi_hash = kmalloc(MSI_HASH_BUCKETS * sizeof(*msi_hash), GFP_KERNEL);
131 if (!msi_hash)
132 return -ENOMEM;
133
134 for (i = 0; i < MSI_HASH_BUCKETS; i++)
135 INIT_HLIST_HEAD(&msi_hash[i]);
136 return 0;
137}
138
139void __init zpci_msihash_exit(void)
140{
141 kfree(msi_hash);
142}
diff --git a/arch/s390/pci/pci_sysfs.c b/arch/s390/pci/pci_sysfs.c
index e99a2557f186..cf8a12ff733b 100644
--- a/arch/s390/pci/pci_sysfs.c
+++ b/arch/s390/pci/pci_sysfs.c
@@ -48,11 +48,38 @@ static ssize_t show_pfgid(struct device *dev, struct device_attribute *attr,
48} 48}
49static DEVICE_ATTR(pfgid, S_IRUGO, show_pfgid, NULL); 49static DEVICE_ATTR(pfgid, S_IRUGO, show_pfgid, NULL);
50 50
51static void recover_callback(struct device *dev)
52{
53 struct pci_dev *pdev = to_pci_dev(dev);
54 struct zpci_dev *zdev = get_zdev(pdev);
55 int ret;
56
57 pci_stop_and_remove_bus_device(pdev);
58 ret = zpci_disable_device(zdev);
59 if (ret)
60 return;
61
62 ret = zpci_enable_device(zdev);
63 if (ret)
64 return;
65
66 pci_rescan_bus(zdev->bus);
67}
68
69static ssize_t store_recover(struct device *dev, struct device_attribute *attr,
70 const char *buf, size_t count)
71{
72 int rc = device_schedule_callback(dev, recover_callback);
73 return rc ? rc : count;
74}
75static DEVICE_ATTR(recover, S_IWUSR, NULL, store_recover);
76
51static struct device_attribute *zpci_dev_attrs[] = { 77static struct device_attribute *zpci_dev_attrs[] = {
52 &dev_attr_function_id, 78 &dev_attr_function_id,
53 &dev_attr_function_handle, 79 &dev_attr_function_handle,
54 &dev_attr_pchid, 80 &dev_attr_pchid,
55 &dev_attr_pfgid, 81 &dev_attr_pfgid,
82 &dev_attr_recover,
56 NULL, 83 NULL,
57}; 84};
58 85
diff --git a/drivers/pci/hotplug/Kconfig b/drivers/pci/hotplug/Kconfig
index d85009de713d..0a648af89531 100644
--- a/drivers/pci/hotplug/Kconfig
+++ b/drivers/pci/hotplug/Kconfig
@@ -146,7 +146,7 @@ config HOTPLUG_PCI_SGI
146 When in doubt, say N. 146 When in doubt, say N.
147 147
148config HOTPLUG_PCI_S390 148config HOTPLUG_PCI_S390
149 tristate "System z PCI Hotplug Support" 149 bool "System z PCI Hotplug Support"
150 depends on S390 && 64BIT 150 depends on S390 && 64BIT
151 help 151 help
152 Say Y here if you want to use the System z PCI Hotplug 152 Say Y here if you want to use the System z PCI Hotplug
diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c
index ea3fa90d020a..66e505ca24ef 100644
--- a/drivers/pci/hotplug/s390_pci_hpc.c
+++ b/drivers/pci/hotplug/s390_pci_hpc.c
@@ -79,8 +79,6 @@ static int enable_slot(struct hotplug_slot *hotplug_slot)
79 if (rc) 79 if (rc)
80 goto out_deconfigure; 80 goto out_deconfigure;
81 81
82 slot->zdev->state = ZPCI_FN_STATE_ONLINE;
83
84 pci_scan_slot(slot->zdev->bus, ZPCI_DEVFN); 82 pci_scan_slot(slot->zdev->bus, ZPCI_DEVFN);
85 pci_bus_add_devices(slot->zdev->bus); 83 pci_bus_add_devices(slot->zdev->bus);
86 84
@@ -148,7 +146,7 @@ static struct hotplug_slot_ops s390_hotplug_slot_ops = {
148 .get_adapter_status = get_adapter_status, 146 .get_adapter_status = get_adapter_status,
149}; 147};
150 148
151static int init_pci_slot(struct zpci_dev *zdev) 149int zpci_init_slot(struct zpci_dev *zdev)
152{ 150{
153 struct hotplug_slot *hotplug_slot; 151 struct hotplug_slot *hotplug_slot;
154 struct hotplug_slot_info *info; 152 struct hotplug_slot_info *info;
@@ -202,7 +200,7 @@ error:
202 return -ENOMEM; 200 return -ENOMEM;
203} 201}
204 202
205static void exit_pci_slot(struct zpci_dev *zdev) 203void zpci_exit_slot(struct zpci_dev *zdev)
206{ 204{
207 struct list_head *tmp, *n; 205 struct list_head *tmp, *n;
208 struct slot *slot; 206 struct slot *slot;
@@ -215,60 +213,3 @@ static void exit_pci_slot(struct zpci_dev *zdev)
215 pci_hp_deregister(slot->hotplug_slot); 213 pci_hp_deregister(slot->hotplug_slot);
216 } 214 }
217} 215}
218
219static struct pci_hp_callback_ops hp_ops = {
220 .create_slot = init_pci_slot,
221 .remove_slot = exit_pci_slot,
222};
223
224static void __init init_pci_slots(void)
225{
226 struct zpci_dev *zdev;
227
228 /*
229 * Create a structure for each slot, and register that slot
230 * with the pci_hotplug subsystem.
231 */
232 mutex_lock(&zpci_list_lock);
233 list_for_each_entry(zdev, &zpci_list, entry) {
234 init_pci_slot(zdev);
235 }
236 mutex_unlock(&zpci_list_lock);
237}
238
239static void __exit exit_pci_slots(void)
240{
241 struct list_head *tmp, *n;
242 struct slot *slot;
243
244 /*
245 * Unregister all of our slots with the pci_hotplug subsystem.
246 * Memory will be freed in release_slot() callback after slot's
247 * lifespan is finished.
248 */
249 list_for_each_safe(tmp, n, &s390_hotplug_slot_list) {
250 slot = list_entry(tmp, struct slot, slot_list);
251 list_del(&slot->slot_list);
252 pci_hp_deregister(slot->hotplug_slot);
253 }
254}
255
256static int __init pci_hotplug_s390_init(void)
257{
258 if (!s390_pci_probe)
259 return -EOPNOTSUPP;
260
261 zpci_register_hp_ops(&hp_ops);
262 init_pci_slots();
263
264 return 0;
265}
266
267static void __exit pci_hotplug_s390_exit(void)
268{
269 exit_pci_slots();
270 zpci_deregister_hp_ops();
271}
272
273module_init(pci_hotplug_s390_init);
274module_exit(pci_hotplug_s390_exit);
diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c
index 58bc6eb49de1..2ead7e78c456 100644
--- a/drivers/s390/block/dasd_devmap.c
+++ b/drivers/s390/block/dasd_devmap.c
@@ -930,7 +930,7 @@ dasd_use_raw_store(struct device *dev, struct device_attribute *attr,
930 if (IS_ERR(devmap)) 930 if (IS_ERR(devmap))
931 return PTR_ERR(devmap); 931 return PTR_ERR(devmap);
932 932
933 if ((strict_strtoul(buf, 10, &val) != 0) || val > 1) 933 if ((kstrtoul(buf, 10, &val) != 0) || val > 1)
934 return -EINVAL; 934 return -EINVAL;
935 935
936 spin_lock(&dasd_devmap_lock); 936 spin_lock(&dasd_devmap_lock);
@@ -1225,7 +1225,7 @@ dasd_expires_store(struct device *dev, struct device_attribute *attr,
1225 if (IS_ERR(device)) 1225 if (IS_ERR(device))
1226 return -ENODEV; 1226 return -ENODEV;
1227 1227
1228 if ((strict_strtoul(buf, 10, &val) != 0) || 1228 if ((kstrtoul(buf, 10, &val) != 0) ||
1229 (val > DASD_EXPIRES_MAX) || val == 0) { 1229 (val > DASD_EXPIRES_MAX) || val == 0) {
1230 dasd_put_device(device); 1230 dasd_put_device(device);
1231 return -EINVAL; 1231 return -EINVAL;
@@ -1265,7 +1265,7 @@ dasd_retries_store(struct device *dev, struct device_attribute *attr,
1265 if (IS_ERR(device)) 1265 if (IS_ERR(device))
1266 return -ENODEV; 1266 return -ENODEV;
1267 1267
1268 if ((strict_strtoul(buf, 10, &val) != 0) || 1268 if ((kstrtoul(buf, 10, &val) != 0) ||
1269 (val > DASD_RETRIES_MAX)) { 1269 (val > DASD_RETRIES_MAX)) {
1270 dasd_put_device(device); 1270 dasd_put_device(device);
1271 return -EINVAL; 1271 return -EINVAL;
@@ -1307,7 +1307,7 @@ dasd_timeout_store(struct device *dev, struct device_attribute *attr,
1307 if (IS_ERR(device) || !device->block) 1307 if (IS_ERR(device) || !device->block)
1308 return -ENODEV; 1308 return -ENODEV;
1309 1309
1310 if ((strict_strtoul(buf, 10, &val) != 0) || 1310 if ((kstrtoul(buf, 10, &val) != 0) ||
1311 val > UINT_MAX / HZ) { 1311 val > UINT_MAX / HZ) {
1312 dasd_put_device(device); 1312 dasd_put_device(device);
1313 return -EINVAL; 1313 return -EINVAL;
diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
index e61a6deea3c0..5adb2042e824 100644
--- a/drivers/s390/block/dasd_eckd.c
+++ b/drivers/s390/block/dasd_eckd.c
@@ -85,6 +85,8 @@ MODULE_DEVICE_TABLE(ccw, dasd_eckd_ids);
85 85
86static struct ccw_driver dasd_eckd_driver; /* see below */ 86static struct ccw_driver dasd_eckd_driver; /* see below */
87 87
88static void *rawpadpage;
89
88#define INIT_CQR_OK 0 90#define INIT_CQR_OK 0
89#define INIT_CQR_UNFORMATTED 1 91#define INIT_CQR_UNFORMATTED 1
90#define INIT_CQR_ERROR 2 92#define INIT_CQR_ERROR 2
@@ -3237,18 +3239,26 @@ static struct dasd_ccw_req *dasd_raw_build_cp(struct dasd_device *startdev,
3237 unsigned int seg_len, len_to_track_end; 3239 unsigned int seg_len, len_to_track_end;
3238 unsigned int first_offs; 3240 unsigned int first_offs;
3239 unsigned int cidaw, cplength, datasize; 3241 unsigned int cidaw, cplength, datasize;
3240 sector_t first_trk, last_trk; 3242 sector_t first_trk, last_trk, sectors;
3243 sector_t start_padding_sectors, end_sector_offset, end_padding_sectors;
3241 unsigned int pfx_datasize; 3244 unsigned int pfx_datasize;
3242 3245
3243 /* 3246 /*
3244 * raw track access needs to be mutiple of 64k and on 64k boundary 3247 * raw track access needs to be mutiple of 64k and on 64k boundary
3248 * For read requests we can fix an incorrect alignment by padding
3249 * the request with dummy pages.
3245 */ 3250 */
3246 if ((blk_rq_pos(req) % DASD_RAW_SECTORS_PER_TRACK) != 0) { 3251 start_padding_sectors = blk_rq_pos(req) % DASD_RAW_SECTORS_PER_TRACK;
3247 cqr = ERR_PTR(-EINVAL); 3252 end_sector_offset = (blk_rq_pos(req) + blk_rq_sectors(req)) %
3248 goto out; 3253 DASD_RAW_SECTORS_PER_TRACK;
3249 } 3254 end_padding_sectors = (DASD_RAW_SECTORS_PER_TRACK - end_sector_offset) %
3250 if (((blk_rq_pos(req) + blk_rq_sectors(req)) % 3255 DASD_RAW_SECTORS_PER_TRACK;
3251 DASD_RAW_SECTORS_PER_TRACK) != 0) { 3256 basedev = block->base;
3257 if ((start_padding_sectors || end_padding_sectors) &&
3258 (rq_data_dir(req) == WRITE)) {
3259 DBF_DEV_EVENT(DBF_ERR, basedev,
3260 "raw write not track aligned (%lu,%lu) req %p",
3261 start_padding_sectors, end_padding_sectors, req);
3252 cqr = ERR_PTR(-EINVAL); 3262 cqr = ERR_PTR(-EINVAL);
3253 goto out; 3263 goto out;
3254 } 3264 }
@@ -3258,7 +3268,6 @@ static struct dasd_ccw_req *dasd_raw_build_cp(struct dasd_device *startdev,
3258 DASD_RAW_SECTORS_PER_TRACK; 3268 DASD_RAW_SECTORS_PER_TRACK;
3259 trkcount = last_trk - first_trk + 1; 3269 trkcount = last_trk - first_trk + 1;
3260 first_offs = 0; 3270 first_offs = 0;
3261 basedev = block->base;
3262 3271
3263 if (rq_data_dir(req) == READ) 3272 if (rq_data_dir(req) == READ)
3264 cmd = DASD_ECKD_CCW_READ_TRACK; 3273 cmd = DASD_ECKD_CCW_READ_TRACK;
@@ -3307,12 +3316,26 @@ static struct dasd_ccw_req *dasd_raw_build_cp(struct dasd_device *startdev,
3307 } 3316 }
3308 3317
3309 idaws = (unsigned long *)(cqr->data + pfx_datasize); 3318 idaws = (unsigned long *)(cqr->data + pfx_datasize);
3310
3311 len_to_track_end = 0; 3319 len_to_track_end = 0;
3312 3320 if (start_padding_sectors) {
3321 ccw[-1].flags |= CCW_FLAG_CC;
3322 ccw->cmd_code = cmd;
3323 /* maximum 3390 track size */
3324 ccw->count = 57326;
3325 /* 64k map to one track */
3326 len_to_track_end = 65536 - start_padding_sectors * 512;
3327 ccw->cda = (__u32)(addr_t)idaws;
3328 ccw->flags |= CCW_FLAG_IDA;
3329 ccw->flags |= CCW_FLAG_SLI;
3330 ccw++;
3331 for (sectors = 0; sectors < start_padding_sectors; sectors += 8)
3332 idaws = idal_create_words(idaws, rawpadpage, PAGE_SIZE);
3333 }
3313 rq_for_each_segment(bv, req, iter) { 3334 rq_for_each_segment(bv, req, iter) {
3314 dst = page_address(bv->bv_page) + bv->bv_offset; 3335 dst = page_address(bv->bv_page) + bv->bv_offset;
3315 seg_len = bv->bv_len; 3336 seg_len = bv->bv_len;
3337 if (cmd == DASD_ECKD_CCW_READ_TRACK)
3338 memset(dst, 0, seg_len);
3316 if (!len_to_track_end) { 3339 if (!len_to_track_end) {
3317 ccw[-1].flags |= CCW_FLAG_CC; 3340 ccw[-1].flags |= CCW_FLAG_CC;
3318 ccw->cmd_code = cmd; 3341 ccw->cmd_code = cmd;
@@ -3328,7 +3351,8 @@ static struct dasd_ccw_req *dasd_raw_build_cp(struct dasd_device *startdev,
3328 len_to_track_end -= seg_len; 3351 len_to_track_end -= seg_len;
3329 idaws = idal_create_words(idaws, dst, seg_len); 3352 idaws = idal_create_words(idaws, dst, seg_len);
3330 } 3353 }
3331 3354 for (sectors = 0; sectors < end_padding_sectors; sectors += 8)
3355 idaws = idal_create_words(idaws, rawpadpage, PAGE_SIZE);
3332 if (blk_noretry_request(req) || 3356 if (blk_noretry_request(req) ||
3333 block->base->features & DASD_FEATURE_FAILFAST) 3357 block->base->features & DASD_FEATURE_FAILFAST)
3334 set_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags); 3358 set_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags);
@@ -4479,12 +4503,19 @@ dasd_eckd_init(void)
4479 kfree(dasd_reserve_req); 4503 kfree(dasd_reserve_req);
4480 return -ENOMEM; 4504 return -ENOMEM;
4481 } 4505 }
4506 rawpadpage = (void *)__get_free_page(GFP_KERNEL);
4507 if (!rawpadpage) {
4508 kfree(path_verification_worker);
4509 kfree(dasd_reserve_req);
4510 return -ENOMEM;
4511 }
4482 ret = ccw_driver_register(&dasd_eckd_driver); 4512 ret = ccw_driver_register(&dasd_eckd_driver);
4483 if (!ret) 4513 if (!ret)
4484 wait_for_device_probe(); 4514 wait_for_device_probe();
4485 else { 4515 else {
4486 kfree(path_verification_worker); 4516 kfree(path_verification_worker);
4487 kfree(dasd_reserve_req); 4517 kfree(dasd_reserve_req);
4518 free_page((unsigned long)rawpadpage);
4488 } 4519 }
4489 return ret; 4520 return ret;
4490} 4521}
@@ -4495,6 +4526,7 @@ dasd_eckd_cleanup(void)
4495 ccw_driver_unregister(&dasd_eckd_driver); 4526 ccw_driver_unregister(&dasd_eckd_driver);
4496 kfree(path_verification_worker); 4527 kfree(path_verification_worker);
4497 kfree(dasd_reserve_req); 4528 kfree(dasd_reserve_req);
4529 free_page((unsigned long)rawpadpage);
4498} 4530}
4499 4531
4500module_init(dasd_eckd_init); 4532module_init(dasd_eckd_init);
diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c
index 8d11f773a752..e1e88486b2b4 100644
--- a/drivers/s390/block/dasd_erp.c
+++ b/drivers/s390/block/dasd_erp.c
@@ -124,10 +124,15 @@ dasd_default_erp_action(struct dasd_ccw_req *cqr)
124struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) 124struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr)
125{ 125{
126 int success; 126 int success;
127 unsigned long long startclk, stopclk;
128 struct dasd_device *startdev;
127 129
128 BUG_ON(cqr->refers == NULL || cqr->function == NULL); 130 BUG_ON(cqr->refers == NULL || cqr->function == NULL);
129 131
130 success = cqr->status == DASD_CQR_DONE; 132 success = cqr->status == DASD_CQR_DONE;
133 startclk = cqr->startclk;
134 stopclk = cqr->stopclk;
135 startdev = cqr->startdev;
131 136
132 /* free all ERPs - but NOT the original cqr */ 137 /* free all ERPs - but NOT the original cqr */
133 while (cqr->refers != NULL) { 138 while (cqr->refers != NULL) {
@@ -142,6 +147,9 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr)
142 } 147 }
143 148
144 /* set corresponding status to original cqr */ 149 /* set corresponding status to original cqr */
150 cqr->startclk = startclk;
151 cqr->stopclk = stopclk;
152 cqr->startdev = startdev;
145 if (success) 153 if (success)
146 cqr->status = DASD_CQR_DONE; 154 cqr->status = DASD_CQR_DONE;
147 else { 155 else {
@@ -160,11 +168,13 @@ dasd_log_sense(struct dasd_ccw_req *cqr, struct irb *irb)
160 168
161 device = cqr->startdev; 169 device = cqr->startdev;
162 if (cqr->intrc == -ETIMEDOUT) { 170 if (cqr->intrc == -ETIMEDOUT) {
163 dev_err(&device->cdev->dev, "cqr %p timeout error", cqr); 171 dev_err(&device->cdev->dev,
172 "A timeout error occurred for cqr %p", cqr);
164 return; 173 return;
165 } 174 }
166 if (cqr->intrc == -ENOLINK) { 175 if (cqr->intrc == -ENOLINK) {
167 dev_err(&device->cdev->dev, "cqr %p transport error", cqr); 176 dev_err(&device->cdev->dev,
177 "A transport error occurred for cqr %p", cqr);
168 return; 178 return;
169 } 179 }
170 /* dump sense data */ 180 /* dump sense data */
diff --git a/drivers/s390/char/sclp_config.c b/drivers/s390/char/sclp_config.c
index 444d36183a25..944156207477 100644
--- a/drivers/s390/char/sclp_config.c
+++ b/drivers/s390/char/sclp_config.c
@@ -32,7 +32,7 @@ static void sclp_cpu_capability_notify(struct work_struct *work)
32 struct device *dev; 32 struct device *dev;
33 33
34 s390_adjust_jiffies(); 34 s390_adjust_jiffies();
35 pr_warning("cpu capability changed.\n"); 35 pr_info("CPU capability may have changed\n");
36 get_online_cpus(); 36 get_online_cpus();
37 for_each_online_cpu(cpu) { 37 for_each_online_cpu(cpu) {
38 dev = get_cpu_device(cpu); 38 dev = get_cpu_device(cpu);
diff --git a/drivers/s390/cio/airq.c b/drivers/s390/cio/airq.c
index 91edbd7ee806..d028fd800c9c 100644
--- a/drivers/s390/cio/airq.c
+++ b/drivers/s390/cio/airq.c
@@ -81,15 +81,185 @@ void unregister_adapter_interrupt(struct airq_struct *airq)
81} 81}
82EXPORT_SYMBOL(unregister_adapter_interrupt); 82EXPORT_SYMBOL(unregister_adapter_interrupt);
83 83
84void do_adapter_IO(u8 isc) 84static irqreturn_t do_airq_interrupt(int irq, void *dummy)
85{ 85{
86 struct tpi_info *tpi_info;
86 struct airq_struct *airq; 87 struct airq_struct *airq;
87 struct hlist_head *head; 88 struct hlist_head *head;
88 89
89 head = &airq_lists[isc]; 90 __this_cpu_write(s390_idle.nohz_delay, 1);
91 tpi_info = (struct tpi_info *) &get_irq_regs()->int_code;
92 head = &airq_lists[tpi_info->isc];
90 rcu_read_lock(); 93 rcu_read_lock();
91 hlist_for_each_entry_rcu(airq, head, list) 94 hlist_for_each_entry_rcu(airq, head, list)
92 if ((*airq->lsi_ptr & airq->lsi_mask) != 0) 95 if ((*airq->lsi_ptr & airq->lsi_mask) != 0)
93 airq->handler(airq); 96 airq->handler(airq);
94 rcu_read_unlock(); 97 rcu_read_unlock();
98
99 return IRQ_HANDLED;
100}
101
102static struct irqaction airq_interrupt = {
103 .name = "AIO",
104 .handler = do_airq_interrupt,
105};
106
107void __init init_airq_interrupts(void)
108{
109 irq_set_chip_and_handler(THIN_INTERRUPT,
110 &dummy_irq_chip, handle_percpu_irq);
111 setup_irq(THIN_INTERRUPT, &airq_interrupt);
112}
113
114/**
115 * airq_iv_create - create an interrupt vector
116 * @bits: number of bits in the interrupt vector
117 * @flags: allocation flags
118 *
119 * Returns a pointer to an interrupt vector structure
120 */
121struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags)
122{
123 struct airq_iv *iv;
124 unsigned long size;
125
126 iv = kzalloc(sizeof(*iv), GFP_KERNEL);
127 if (!iv)
128 goto out;
129 iv->bits = bits;
130 size = BITS_TO_LONGS(bits) * sizeof(unsigned long);
131 iv->vector = kzalloc(size, GFP_KERNEL);
132 if (!iv->vector)
133 goto out_free;
134 if (flags & AIRQ_IV_ALLOC) {
135 iv->avail = kmalloc(size, GFP_KERNEL);
136 if (!iv->avail)
137 goto out_free;
138 memset(iv->avail, 0xff, size);
139 iv->end = 0;
140 } else
141 iv->end = bits;
142 if (flags & AIRQ_IV_BITLOCK) {
143 iv->bitlock = kzalloc(size, GFP_KERNEL);
144 if (!iv->bitlock)
145 goto out_free;
146 }
147 if (flags & AIRQ_IV_PTR) {
148 size = bits * sizeof(unsigned long);
149 iv->ptr = kzalloc(size, GFP_KERNEL);
150 if (!iv->ptr)
151 goto out_free;
152 }
153 if (flags & AIRQ_IV_DATA) {
154 size = bits * sizeof(unsigned int);
155 iv->data = kzalloc(size, GFP_KERNEL);
156 if (!iv->data)
157 goto out_free;
158 }
159 spin_lock_init(&iv->lock);
160 return iv;
161
162out_free:
163 kfree(iv->ptr);
164 kfree(iv->bitlock);
165 kfree(iv->avail);
166 kfree(iv->vector);
167 kfree(iv);
168out:
169 return NULL;
170}
171EXPORT_SYMBOL(airq_iv_create);
172
173/**
174 * airq_iv_release - release an interrupt vector
175 * @iv: pointer to interrupt vector structure
176 */
177void airq_iv_release(struct airq_iv *iv)
178{
179 kfree(iv->data);
180 kfree(iv->ptr);
181 kfree(iv->bitlock);
182 kfree(iv->vector);
183 kfree(iv->avail);
184 kfree(iv);
185}
186EXPORT_SYMBOL(airq_iv_release);
187
188/**
189 * airq_iv_alloc_bit - allocate an irq bit from an interrupt vector
190 * @iv: pointer to an interrupt vector structure
191 *
192 * Returns the bit number of the allocated irq, or -1UL if no bit
193 * is available or the AIRQ_IV_ALLOC flag has not been specified
194 */
195unsigned long airq_iv_alloc_bit(struct airq_iv *iv)
196{
197 const unsigned long be_to_le = BITS_PER_LONG - 1;
198 unsigned long bit;
199
200 if (!iv->avail)
201 return -1UL;
202 spin_lock(&iv->lock);
203 bit = find_first_bit_left(iv->avail, iv->bits);
204 if (bit < iv->bits) {
205 clear_bit(bit ^ be_to_le, iv->avail);
206 if (bit >= iv->end)
207 iv->end = bit + 1;
208 } else
209 bit = -1UL;
210 spin_unlock(&iv->lock);
211 return bit;
212
213}
214EXPORT_SYMBOL(airq_iv_alloc_bit);
215
216/**
217 * airq_iv_free_bit - free an irq bit of an interrupt vector
218 * @iv: pointer to interrupt vector structure
219 * @bit: number of the irq bit to free
220 */
221void airq_iv_free_bit(struct airq_iv *iv, unsigned long bit)
222{
223 const unsigned long be_to_le = BITS_PER_LONG - 1;
224
225 if (!iv->avail)
226 return;
227 spin_lock(&iv->lock);
228 /* Clear (possibly left over) interrupt bit */
229 clear_bit(bit ^ be_to_le, iv->vector);
230 /* Make the bit position available again */
231 set_bit(bit ^ be_to_le, iv->avail);
232 if (bit == iv->end - 1) {
233 /* Find new end of bit-field */
234 while (--iv->end > 0)
235 if (!test_bit((iv->end - 1) ^ be_to_le, iv->avail))
236 break;
237 }
238 spin_unlock(&iv->lock);
239}
240EXPORT_SYMBOL(airq_iv_free_bit);
241
242/**
243 * airq_iv_scan - scan interrupt vector for non-zero bits
244 * @iv: pointer to interrupt vector structure
245 * @start: bit number to start the search
246 * @end: bit number to end the search
247 *
248 * Returns the bit number of the next non-zero interrupt bit, or
249 * -1UL if the scan completed without finding any more any non-zero bits.
250 */
251unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start,
252 unsigned long end)
253{
254 const unsigned long be_to_le = BITS_PER_LONG - 1;
255 unsigned long bit;
256
257 /* Find non-zero bit starting from 'ivs->next'. */
258 bit = find_next_bit_left(iv->vector, end, start);
259 if (bit >= end)
260 return -1UL;
261 /* Clear interrupt bit (find left uses big-endian bit numbers) */
262 clear_bit(bit ^ be_to_le, iv->vector);
263 return bit;
95} 264}
265EXPORT_SYMBOL(airq_iv_scan);
diff --git a/drivers/s390/cio/ccwgroup.c b/drivers/s390/cio/ccwgroup.c
index 84846c2b96d3..959135a01847 100644
--- a/drivers/s390/cio/ccwgroup.c
+++ b/drivers/s390/cio/ccwgroup.c
@@ -137,7 +137,7 @@ static ssize_t ccwgroup_online_store(struct device *dev,
137 if (!try_module_get(gdrv->driver.owner)) 137 if (!try_module_get(gdrv->driver.owner))
138 return -EINVAL; 138 return -EINVAL;
139 139
140 ret = strict_strtoul(buf, 0, &value); 140 ret = kstrtoul(buf, 0, &value);
141 if (ret) 141 if (ret)
142 goto out; 142 goto out;
143 143
diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c
index 4eeb4a6bf207..d7da67a31c77 100644
--- a/drivers/s390/cio/cio.c
+++ b/drivers/s390/cio/cio.c
@@ -561,37 +561,23 @@ out:
561} 561}
562 562
563/* 563/*
564 * do_IRQ() handles all normal I/O device IRQ's (the special 564 * do_cio_interrupt() handles all normal I/O device IRQ's
565 * SMP cross-CPU interrupts have their own specific
566 * handlers).
567 *
568 */ 565 */
569void __irq_entry do_IRQ(struct pt_regs *regs) 566static irqreturn_t do_cio_interrupt(int irq, void *dummy)
570{ 567{
571 struct tpi_info *tpi_info = (struct tpi_info *) &regs->int_code; 568 struct tpi_info *tpi_info;
572 struct subchannel *sch; 569 struct subchannel *sch;
573 struct irb *irb; 570 struct irb *irb;
574 struct pt_regs *old_regs;
575 571
576 old_regs = set_irq_regs(regs);
577 irq_enter();
578 __this_cpu_write(s390_idle.nohz_delay, 1); 572 __this_cpu_write(s390_idle.nohz_delay, 1);
579 if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) 573 tpi_info = (struct tpi_info *) &get_irq_regs()->int_code;
580 /* Serve timer interrupts first. */
581 clock_comparator_work();
582
583 kstat_incr_irqs_this_cpu(IO_INTERRUPT, NULL);
584 irb = (struct irb *) &S390_lowcore.irb; 574 irb = (struct irb *) &S390_lowcore.irb;
585 if (tpi_info->adapter_IO) {
586 do_adapter_IO(tpi_info->isc);
587 goto out;
588 }
589 sch = (struct subchannel *)(unsigned long) tpi_info->intparm; 575 sch = (struct subchannel *)(unsigned long) tpi_info->intparm;
590 if (!sch) { 576 if (!sch) {
591 /* Clear pending interrupt condition. */ 577 /* Clear pending interrupt condition. */
592 inc_irq_stat(IRQIO_CIO); 578 inc_irq_stat(IRQIO_CIO);
593 tsch(tpi_info->schid, irb); 579 tsch(tpi_info->schid, irb);
594 goto out; 580 return IRQ_HANDLED;
595 } 581 }
596 spin_lock(sch->lock); 582 spin_lock(sch->lock);
597 /* Store interrupt response block to lowcore. */ 583 /* Store interrupt response block to lowcore. */
@@ -606,9 +592,23 @@ void __irq_entry do_IRQ(struct pt_regs *regs)
606 } else 592 } else
607 inc_irq_stat(IRQIO_CIO); 593 inc_irq_stat(IRQIO_CIO);
608 spin_unlock(sch->lock); 594 spin_unlock(sch->lock);
609out: 595
610 irq_exit(); 596 return IRQ_HANDLED;
611 set_irq_regs(old_regs); 597}
598
599static struct irq_desc *irq_desc_io;
600
601static struct irqaction io_interrupt = {
602 .name = "IO",
603 .handler = do_cio_interrupt,
604};
605
606void __init init_cio_interrupts(void)
607{
608 irq_set_chip_and_handler(IO_INTERRUPT,
609 &dummy_irq_chip, handle_percpu_irq);
610 setup_irq(IO_INTERRUPT, &io_interrupt);
611 irq_desc_io = irq_to_desc(IO_INTERRUPT);
612} 612}
613 613
614#ifdef CONFIG_CCW_CONSOLE 614#ifdef CONFIG_CCW_CONSOLE
@@ -635,7 +635,7 @@ void cio_tsch(struct subchannel *sch)
635 local_bh_disable(); 635 local_bh_disable();
636 irq_enter(); 636 irq_enter();
637 } 637 }
638 kstat_incr_irqs_this_cpu(IO_INTERRUPT, NULL); 638 kstat_incr_irqs_this_cpu(IO_INTERRUPT, irq_desc_io);
639 if (sch->driver && sch->driver->irq) 639 if (sch->driver && sch->driver->irq)
640 sch->driver->irq(sch); 640 sch->driver->irq(sch);
641 else 641 else
diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h
index d62f5e7f3cf1..d42f67412bd8 100644
--- a/drivers/s390/cio/cio.h
+++ b/drivers/s390/cio/cio.h
@@ -121,9 +121,6 @@ extern int cio_commit_config(struct subchannel *sch);
121int cio_tm_start_key(struct subchannel *sch, struct tcw *tcw, u8 lpm, u8 key); 121int cio_tm_start_key(struct subchannel *sch, struct tcw *tcw, u8 lpm, u8 key);
122int cio_tm_intrg(struct subchannel *sch); 122int cio_tm_intrg(struct subchannel *sch);
123 123
124void do_adapter_IO(u8 isc);
125void do_IRQ(struct pt_regs *);
126
127/* Use with care. */ 124/* Use with care. */
128#ifdef CONFIG_CCW_CONSOLE 125#ifdef CONFIG_CCW_CONSOLE
129extern struct subchannel *cio_probe_console(void); 126extern struct subchannel *cio_probe_console(void);
diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c
index 4495e0627a40..23054f8fa9fc 100644
--- a/drivers/s390/cio/cmf.c
+++ b/drivers/s390/cio/cmf.c
@@ -1182,7 +1182,7 @@ static ssize_t cmb_enable_store(struct device *dev,
1182 int ret; 1182 int ret;
1183 unsigned long val; 1183 unsigned long val;
1184 1184
1185 ret = strict_strtoul(buf, 16, &val); 1185 ret = kstrtoul(buf, 16, &val);
1186 if (ret) 1186 if (ret)
1187 return ret; 1187 return ret;
1188 1188
diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c
index 1ebe5d3ddebb..8c2cb87bccc5 100644
--- a/drivers/s390/cio/css.c
+++ b/drivers/s390/cio/css.c
@@ -546,7 +546,9 @@ static int slow_eval_unknown_fn(struct subchannel_id schid, void *data)
546 case -ENOMEM: 546 case -ENOMEM:
547 case -EIO: 547 case -EIO:
548 /* These should abort looping */ 548 /* These should abort looping */
549 spin_lock_irq(&slow_subchannel_lock);
549 idset_sch_del_subseq(slow_subchannel_set, schid); 550 idset_sch_del_subseq(slow_subchannel_set, schid);
551 spin_unlock_irq(&slow_subchannel_lock);
550 break; 552 break;
551 default: 553 default:
552 rc = 0; 554 rc = 0;
@@ -740,7 +742,7 @@ css_cm_enable_store(struct device *dev, struct device_attribute *attr,
740 int ret; 742 int ret;
741 unsigned long val; 743 unsigned long val;
742 744
743 ret = strict_strtoul(buf, 16, &val); 745 ret = kstrtoul(buf, 16, &val);
744 if (ret) 746 if (ret)
745 return ret; 747 return ret;
746 mutex_lock(&css->mutex); 748 mutex_lock(&css->mutex);
diff --git a/drivers/s390/cio/css.h b/drivers/s390/cio/css.h
index b1de60335238..29351321bad6 100644
--- a/drivers/s390/cio/css.h
+++ b/drivers/s390/cio/css.h
@@ -130,8 +130,6 @@ struct channel_subsystem {
130 130
131extern struct channel_subsystem *channel_subsystems[]; 131extern struct channel_subsystem *channel_subsystems[];
132 132
133void channel_subsystem_reinit(void);
134
135/* Helper functions to build lists for the slow path. */ 133/* Helper functions to build lists for the slow path. */
136void css_schedule_eval(struct subchannel_id schid); 134void css_schedule_eval(struct subchannel_id schid);
137void css_schedule_eval_all(void); 135void css_schedule_eval_all(void);
diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
index 1ab5f6c36d9b..e4a7ab2bb629 100644
--- a/drivers/s390/cio/device.c
+++ b/drivers/s390/cio/device.c
@@ -564,7 +564,7 @@ static ssize_t online_store (struct device *dev, struct device_attribute *attr,
564 ret = 0; 564 ret = 0;
565 } else { 565 } else {
566 force = 0; 566 force = 0;
567 ret = strict_strtoul(buf, 16, &i); 567 ret = kstrtoul(buf, 16, &i);
568 } 568 }
569 if (ret) 569 if (ret)
570 goto out; 570 goto out;
diff --git a/drivers/s390/net/qeth_l3_sys.c b/drivers/s390/net/qeth_l3_sys.c
index d1c8025b0b03..adef5f5de118 100644
--- a/drivers/s390/net/qeth_l3_sys.c
+++ b/drivers/s390/net/qeth_l3_sys.c
@@ -208,7 +208,7 @@ static ssize_t qeth_l3_dev_sniffer_store(struct device *dev,
208 goto out; 208 goto out;
209 } 209 }
210 210
211 rc = strict_strtoul(buf, 16, &i); 211 rc = kstrtoul(buf, 16, &i);
212 if (rc) { 212 if (rc) {
213 rc = -EINVAL; 213 rc = -EINVAL;
214 goto out; 214 goto out;
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index 0807ddf97b05..f330d28e4d0e 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -208,10 +208,6 @@ static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
208#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 208#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
209#endif 209#endif
210 210
211#ifndef __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG
212#define page_test_and_clear_young(pfn) (0)
213#endif
214
215#ifndef __HAVE_ARCH_PGD_OFFSET_GATE 211#ifndef __HAVE_ARCH_PGD_OFFSET_GATE
216#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr) 212#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
217#endif 213#endif
diff --git a/mm/rmap.c b/mm/rmap.c
index b2e29acd7e3d..07748e68b729 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -873,9 +873,6 @@ int page_referenced(struct page *page,
873 vm_flags); 873 vm_flags);
874 if (we_locked) 874 if (we_locked)
875 unlock_page(page); 875 unlock_page(page);
876
877 if (page_test_and_clear_young(page_to_pfn(page)))
878 referenced++;
879 } 876 }
880out: 877out:
881 return referenced; 878 return referenced;
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-23 07:08:45 -0400