diff options
Diffstat (limited to 'arch/x86/mm/kmmio.c')
-rw-r--r-- | arch/x86/mm/kmmio.c | 104 |
1 files changed, 61 insertions, 43 deletions
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c index 50dc802a1c46..16ccbd77917f 100644 --- a/arch/x86/mm/kmmio.c +++ b/arch/x86/mm/kmmio.c | |||
@@ -32,7 +32,7 @@ struct kmmio_fault_page { | |||
32 | struct list_head list; | 32 | struct list_head list; |
33 | struct kmmio_fault_page *release_next; | 33 | struct kmmio_fault_page *release_next; |
34 | unsigned long page; /* location of the fault page */ | 34 | unsigned long page; /* location of the fault page */ |
35 | bool old_presence; /* page presence prior to arming */ | 35 | pteval_t old_presence; /* page presence prior to arming */ |
36 | bool armed; | 36 | bool armed; |
37 | 37 | ||
38 | /* | 38 | /* |
@@ -97,60 +97,62 @@ static struct kmmio_probe *get_kmmio_probe(unsigned long addr) | |||
97 | static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page) | 97 | static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page) |
98 | { | 98 | { |
99 | struct list_head *head; | 99 | struct list_head *head; |
100 | struct kmmio_fault_page *p; | 100 | struct kmmio_fault_page *f; |
101 | 101 | ||
102 | page &= PAGE_MASK; | 102 | page &= PAGE_MASK; |
103 | head = kmmio_page_list(page); | 103 | head = kmmio_page_list(page); |
104 | list_for_each_entry_rcu(p, head, list) { | 104 | list_for_each_entry_rcu(f, head, list) { |
105 | if (p->page == page) | 105 | if (f->page == page) |
106 | return p; | 106 | return f; |
107 | } | 107 | } |
108 | return NULL; | 108 | return NULL; |
109 | } | 109 | } |
110 | 110 | ||
111 | static void set_pmd_presence(pmd_t *pmd, bool present, bool *old) | 111 | static void clear_pmd_presence(pmd_t *pmd, bool clear, pmdval_t *old) |
112 | { | 112 | { |
113 | pmdval_t v = pmd_val(*pmd); | 113 | pmdval_t v = pmd_val(*pmd); |
114 | *old = !!(v & _PAGE_PRESENT); | 114 | if (clear) { |
115 | v &= ~_PAGE_PRESENT; | 115 | *old = v & _PAGE_PRESENT; |
116 | if (present) | 116 | v &= ~_PAGE_PRESENT; |
117 | v |= _PAGE_PRESENT; | 117 | } else /* presume this has been called with clear==true previously */ |
118 | v |= *old; | ||
118 | set_pmd(pmd, __pmd(v)); | 119 | set_pmd(pmd, __pmd(v)); |
119 | } | 120 | } |
120 | 121 | ||
121 | static void set_pte_presence(pte_t *pte, bool present, bool *old) | 122 | static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old) |
122 | { | 123 | { |
123 | pteval_t v = pte_val(*pte); | 124 | pteval_t v = pte_val(*pte); |
124 | *old = !!(v & _PAGE_PRESENT); | 125 | if (clear) { |
125 | v &= ~_PAGE_PRESENT; | 126 | *old = v & _PAGE_PRESENT; |
126 | if (present) | 127 | v &= ~_PAGE_PRESENT; |
127 | v |= _PAGE_PRESENT; | 128 | } else /* presume this has been called with clear==true previously */ |
129 | v |= *old; | ||
128 | set_pte_atomic(pte, __pte(v)); | 130 | set_pte_atomic(pte, __pte(v)); |
129 | } | 131 | } |
130 | 132 | ||
131 | static int set_page_presence(unsigned long addr, bool present, bool *old) | 133 | static int clear_page_presence(struct kmmio_fault_page *f, bool clear) |
132 | { | 134 | { |
133 | unsigned int level; | 135 | unsigned int level; |
134 | pte_t *pte = lookup_address(addr, &level); | 136 | pte_t *pte = lookup_address(f->page, &level); |
135 | 137 | ||
136 | if (!pte) { | 138 | if (!pte) { |
137 | pr_err("kmmio: no pte for page 0x%08lx\n", addr); | 139 | pr_err("kmmio: no pte for page 0x%08lx\n", f->page); |
138 | return -1; | 140 | return -1; |
139 | } | 141 | } |
140 | 142 | ||
141 | switch (level) { | 143 | switch (level) { |
142 | case PG_LEVEL_2M: | 144 | case PG_LEVEL_2M: |
143 | set_pmd_presence((pmd_t *)pte, present, old); | 145 | clear_pmd_presence((pmd_t *)pte, clear, &f->old_presence); |
144 | break; | 146 | break; |
145 | case PG_LEVEL_4K: | 147 | case PG_LEVEL_4K: |
146 | set_pte_presence(pte, present, old); | 148 | clear_pte_presence(pte, clear, &f->old_presence); |
147 | break; | 149 | break; |
148 | default: | 150 | default: |
149 | pr_err("kmmio: unexpected page level 0x%x.\n", level); | 151 | pr_err("kmmio: unexpected page level 0x%x.\n", level); |
150 | return -1; | 152 | return -1; |
151 | } | 153 | } |
152 | 154 | ||
153 | __flush_tlb_one(addr); | 155 | __flush_tlb_one(f->page); |
154 | return 0; | 156 | return 0; |
155 | } | 157 | } |
156 | 158 | ||
@@ -171,9 +173,9 @@ static int arm_kmmio_fault_page(struct kmmio_fault_page *f) | |||
171 | WARN_ONCE(f->armed, KERN_ERR "kmmio page already armed.\n"); | 173 | WARN_ONCE(f->armed, KERN_ERR "kmmio page already armed.\n"); |
172 | if (f->armed) { | 174 | if (f->armed) { |
173 | pr_warning("kmmio double-arm: page 0x%08lx, ref %d, old %d\n", | 175 | pr_warning("kmmio double-arm: page 0x%08lx, ref %d, old %d\n", |
174 | f->page, f->count, f->old_presence); | 176 | f->page, f->count, !!f->old_presence); |
175 | } | 177 | } |
176 | ret = set_page_presence(f->page, false, &f->old_presence); | 178 | ret = clear_page_presence(f, true); |
177 | WARN_ONCE(ret < 0, KERN_ERR "kmmio arming 0x%08lx failed.\n", f->page); | 179 | WARN_ONCE(ret < 0, KERN_ERR "kmmio arming 0x%08lx failed.\n", f->page); |
178 | f->armed = true; | 180 | f->armed = true; |
179 | return ret; | 181 | return ret; |
@@ -182,8 +184,7 @@ static int arm_kmmio_fault_page(struct kmmio_fault_page *f) | |||
182 | /** Restore the given page to saved presence state. */ | 184 | /** Restore the given page to saved presence state. */ |
183 | static void disarm_kmmio_fault_page(struct kmmio_fault_page *f) | 185 | static void disarm_kmmio_fault_page(struct kmmio_fault_page *f) |
184 | { | 186 | { |
185 | bool tmp; | 187 | int ret = clear_page_presence(f, false); |
186 | int ret = set_page_presence(f->page, f->old_presence, &tmp); | ||
187 | WARN_ONCE(ret < 0, | 188 | WARN_ONCE(ret < 0, |
188 | KERN_ERR "kmmio disarming 0x%08lx failed.\n", f->page); | 189 | KERN_ERR "kmmio disarming 0x%08lx failed.\n", f->page); |
189 | f->armed = false; | 190 | f->armed = false; |
@@ -310,7 +311,12 @@ static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs) | |||
310 | struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx); | 311 | struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx); |
311 | 312 | ||
312 | if (!ctx->active) { | 313 | if (!ctx->active) { |
313 | pr_debug("kmmio: spurious debug trap on CPU %d.\n", | 314 | /* |
315 | * debug traps without an active context are due to either | ||
316 | * something external causing them (f.e. using a debugger while | ||
317 | * mmio tracing enabled), or erroneous behaviour | ||
318 | */ | ||
319 | pr_warning("kmmio: unexpected debug trap on CPU %d.\n", | ||
314 | smp_processor_id()); | 320 | smp_processor_id()); |
315 | goto out; | 321 | goto out; |
316 | } | 322 | } |
@@ -439,12 +445,12 @@ static void rcu_free_kmmio_fault_pages(struct rcu_head *head) | |||
439 | head, | 445 | head, |
440 | struct kmmio_delayed_release, | 446 | struct kmmio_delayed_release, |
441 | rcu); | 447 | rcu); |
442 | struct kmmio_fault_page *p = dr->release_list; | 448 | struct kmmio_fault_page *f = dr->release_list; |
443 | while (p) { | 449 | while (f) { |
444 | struct kmmio_fault_page *next = p->release_next; | 450 | struct kmmio_fault_page *next = f->release_next; |
445 | BUG_ON(p->count); | 451 | BUG_ON(f->count); |
446 | kfree(p); | 452 | kfree(f); |
447 | p = next; | 453 | f = next; |
448 | } | 454 | } |
449 | kfree(dr); | 455 | kfree(dr); |
450 | } | 456 | } |
@@ -453,19 +459,19 @@ static void remove_kmmio_fault_pages(struct rcu_head *head) | |||
453 | { | 459 | { |
454 | struct kmmio_delayed_release *dr = | 460 | struct kmmio_delayed_release *dr = |
455 | container_of(head, struct kmmio_delayed_release, rcu); | 461 | container_of(head, struct kmmio_delayed_release, rcu); |
456 | struct kmmio_fault_page *p = dr->release_list; | 462 | struct kmmio_fault_page *f = dr->release_list; |
457 | struct kmmio_fault_page **prevp = &dr->release_list; | 463 | struct kmmio_fault_page **prevp = &dr->release_list; |
458 | unsigned long flags; | 464 | unsigned long flags; |
459 | 465 | ||
460 | spin_lock_irqsave(&kmmio_lock, flags); | 466 | spin_lock_irqsave(&kmmio_lock, flags); |
461 | while (p) { | 467 | while (f) { |
462 | if (!p->count) { | 468 | if (!f->count) { |
463 | list_del_rcu(&p->list); | 469 | list_del_rcu(&f->list); |
464 | prevp = &p->release_next; | 470 | prevp = &f->release_next; |
465 | } else { | 471 | } else { |
466 | *prevp = p->release_next; | 472 | *prevp = f->release_next; |
467 | } | 473 | } |
468 | p = p->release_next; | 474 | f = f->release_next; |
469 | } | 475 | } |
470 | spin_unlock_irqrestore(&kmmio_lock, flags); | 476 | spin_unlock_irqrestore(&kmmio_lock, flags); |
471 | 477 | ||
@@ -528,8 +534,8 @@ void unregister_kmmio_probe(struct kmmio_probe *p) | |||
528 | } | 534 | } |
529 | EXPORT_SYMBOL(unregister_kmmio_probe); | 535 | EXPORT_SYMBOL(unregister_kmmio_probe); |
530 | 536 | ||
531 | static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val, | 537 | static int |
532 | void *args) | 538 | kmmio_die_notifier(struct notifier_block *nb, unsigned long val, void *args) |
533 | { | 539 | { |
534 | struct die_args *arg = args; | 540 | struct die_args *arg = args; |
535 | 541 | ||
@@ -544,11 +550,23 @@ static struct notifier_block nb_die = { | |||
544 | .notifier_call = kmmio_die_notifier | 550 | .notifier_call = kmmio_die_notifier |
545 | }; | 551 | }; |
546 | 552 | ||
547 | static int __init init_kmmio(void) | 553 | int kmmio_init(void) |
548 | { | 554 | { |
549 | int i; | 555 | int i; |
556 | |||
550 | for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++) | 557 | for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++) |
551 | INIT_LIST_HEAD(&kmmio_page_table[i]); | 558 | INIT_LIST_HEAD(&kmmio_page_table[i]); |
559 | |||
552 | return register_die_notifier(&nb_die); | 560 | return register_die_notifier(&nb_die); |
553 | } | 561 | } |
554 | fs_initcall(init_kmmio); /* should be before device_initcall() */ | 562 | |
563 | void kmmio_cleanup(void) | ||
564 | { | ||
565 | int i; | ||
566 | |||
567 | unregister_die_notifier(&nb_die); | ||
568 | for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++) { | ||
569 | WARN_ONCE(!list_empty(&kmmio_page_table[i]), | ||
570 | KERN_ERR "kmmio_page_table not empty at cleanup, any further tracing will leak memory.\n"); | ||
571 | } | ||
572 | } | ||