1 files changed, 54 insertions, 49 deletions

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index d237d3350a99..f25aa98a94df 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -81,6 +81,11 @@ unsigned int halt_poll_ns_grow = 2;
 module_param(halt_poll_ns_grow, uint, 0644);
 EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
 
+/* The start value to grow halt_poll_ns from */
+unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
+module_param(halt_poll_ns_grow_start, uint, 0644);
+EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);
+
 /* Default resets per-vcpu halt_poll_ns . */
 unsigned int halt_poll_ns_shrink;
 module_param(halt_poll_ns_shrink, uint, 0644);
@@ -525,7 +530,7 @@ static struct kvm_memslots *kvm_alloc_memslots(void)
         int i;
         struct kvm_memslots *slots;
 
-        slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
+        slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
         if (!slots)
                 return NULL;
 
@@ -601,12 +606,12 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
 
         kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
                                          sizeof(*kvm->debugfs_stat_data),
-                                         GFP_KERNEL);
+                                         GFP_KERNEL_ACCOUNT);
         if (!kvm->debugfs_stat_data)
                 return -ENOMEM;
 
         for (p = debugfs_entries; p->name; p++) {
-                stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL);
+                stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
                 if (!stat_data)
                         return -ENOMEM;
 
@@ -656,12 +661,8 @@ static struct kvm *kvm_create_vm(unsigned long type)
                 struct kvm_memslots *slots = kvm_alloc_memslots();
                 if (!slots)
                         goto out_err_no_srcu;
-                /*
-                 * Generations must be different for each address space.
-                 * Init kvm generation close to the maximum to easily test the
-                 * code of handling generation number wrap-around.
-                 */
-                slots->generation = i * 2 - 150;
+                /* Generations must be different for each address space. */
+                slots->generation = i;
                 rcu_assign_pointer(kvm->memslots[i], slots);
         }
 
@@ -671,7 +672,7 @@ static struct kvm *kvm_create_vm(unsigned long type)
                 goto out_err_no_irq_srcu;
         for (i = 0; i < KVM_NR_BUSES; i++) {
                 rcu_assign_pointer(kvm->buses[i],
-                        kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL));
+                        kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
                 if (!kvm->buses[i])
                         goto out_err;
         }
@@ -789,7 +790,7 @@ static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
 {
         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
 
-        memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL);
+        memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
         if (!memslot->dirty_bitmap)
                 return -ENOMEM;
 
@@ -874,31 +875,34 @@ static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
                 int as_id, struct kvm_memslots *slots)
 {
         struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
+        u64 gen = old_memslots->generation;
 
-        /*
-         * Set the low bit in the generation, which disables SPTE caching
-         * until the end of synchronize_srcu_expedited.
-         */
-        WARN_ON(old_memslots->generation & 1);
-        slots->generation = old_memslots->generation + 1;
+        WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
+        slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
 
         rcu_assign_pointer(kvm->memslots[as_id], slots);
         synchronize_srcu_expedited(&kvm->srcu);
 
         /*
-         * Increment the new memslot generation a second time. This prevents
-         * vm exits that race with memslot updates from caching a memslot
-         * generation that will (potentially) be valid forever.
-         *
+         * Increment the new memslot generation a second time, dropping the
+         * update in-progress flag and incrementing then generation based on
+         * the number of address spaces.  This provides a unique and easily
+         * identifiable generation number while the memslots are in flux.
+         */
+        gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
+
+        /*
          * Generations must be unique even across address spaces.  We do not need
          * a global counter for that, instead the generation space is evenly split
          * across address spaces.  For example, with two address spaces, address
-         * space 0 will use generations 0, 4, 8, ... while * address space 1 will
-         * use generations 2, 6, 10, 14, ...
+         * space 0 will use generations 0, 2, 4, ... while address space 1 will
+         * use generations 1, 3, 5, ...
          */
-        slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1;
+        gen += KVM_ADDRESS_SPACE_NUM;
+
+        kvm_arch_memslots_updated(kvm, gen);
 
-        kvm_arch_memslots_updated(kvm, slots);
+        slots->generation = gen;
 
         return old_memslots;
 }
@@ -1018,7 +1022,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
                         goto out_free;
         }
 
-        slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
+        slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
         if (!slots)
                 goto out_free;
         memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
@@ -1201,11 +1205,9 @@ int kvm_get_dirty_log_protect(struct kvm *kvm,
                         mask = xchg(&dirty_bitmap[i], 0);
                         dirty_bitmap_buffer[i] = mask;
 
-                        if (mask) {
-                                offset = i * BITS_PER_LONG;
-                                kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
-                                                                        offset, mask);
-                        }
+                        offset = i * BITS_PER_LONG;
+                        kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
+                                                                offset, mask);
                 }
                 spin_unlock(&kvm->mmu_lock);
         }
@@ -2185,20 +2187,23 @@ void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
 
 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
 {
-        unsigned int old, val, grow;
+        unsigned int old, val, grow, grow_start;
 
         old = val = vcpu->halt_poll_ns;
+        grow_start = READ_ONCE(halt_poll_ns_grow_start);
         grow = READ_ONCE(halt_poll_ns_grow);
-        /* 10us base */
-        if (val == 0 && grow)
-                val = 10000;
-        else
-                val *= grow;
+        if (!grow)
+                goto out;
+
+        val *= grow;
+        if (val < grow_start)
+                val = grow_start;
 
         if (val > halt_poll_ns)
                 val = halt_poll_ns;
 
         vcpu->halt_poll_ns = val;
+out:
         trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
 }
 
@@ -2683,7 +2688,7 @@ static long kvm_vcpu_ioctl(struct file *filp,
                 struct kvm_regs *kvm_regs;
 
                 r = -ENOMEM;
-                kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
+                kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
                 if (!kvm_regs)
                         goto out;
                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
@@ -2711,7 +2716,8 @@ out_free1:
                 break;
         }
         case KVM_GET_SREGS: {
-                kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
+                kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
+                                    GFP_KERNEL_ACCOUNT);
                 r = -ENOMEM;
                 if (!kvm_sregs)
                         goto out;
@@ -2803,7 +2809,7 @@ out_free1:
                 break;
         }
         case KVM_GET_FPU: {
-                fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
+                fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
                 r = -ENOMEM;
                 if (!fpu)
                         goto out;
@@ -2980,7 +2986,7 @@ static int kvm_ioctl_create_device(struct kvm *kvm,
         if (test)
                 return 0;
 
-        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+        dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
         if (!dev)
                 return -ENOMEM;
 
@@ -3625,6 +3631,7 @@ int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
         r = __kvm_io_bus_write(vcpu, bus, &range, val);
         return r < 0 ? r : 0;
 }
+EXPORT_SYMBOL_GPL(kvm_io_bus_write);
 
 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
@@ -3675,7 +3682,6 @@ static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
 
         return -EOPNOTSUPP;
 }
-EXPORT_SYMBOL_GPL(kvm_io_bus_write);
 
 /* kvm_io_bus_read - called under kvm->slots_lock */
 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
@@ -3697,7 +3703,6 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
         return r < 0 ? r : 0;
 }
 
-
 /* Caller must hold slots_lock. */
 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
                             int len, struct kvm_io_device *dev)
@@ -3714,8 +3719,8 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
         if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
                 return -ENOSPC;
 
-        new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) *
-                          sizeof(struct kvm_io_range)), GFP_KERNEL);
+        new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
+                          GFP_KERNEL_ACCOUNT);
         if (!new_bus)
                 return -ENOMEM;
 
@@ -3760,8 +3765,8 @@ void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
         if (i == bus->dev_count)
                 return;
 
-        new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) *
-                          sizeof(struct kvm_io_range)), GFP_KERNEL);
+        new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
+                          GFP_KERNEL_ACCOUNT);
         if (!new_bus)  {
                 pr_err("kvm: failed to shrink bus, removing it completely\n");
                 goto broken;
@@ -4029,7 +4034,7 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
         active = kvm_active_vms;
         spin_unlock(&kvm_lock);
 
-        env = kzalloc(sizeof(*env), GFP_KERNEL);
+        env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
         if (!env)
                 return;
 
@@ -4045,7 +4050,7 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
         add_uevent_var(env, "PID=%d", kvm->userspace_pid);
 
         if (!IS_ERR_OR_NULL(kvm->debugfs_dentry)) {
-                char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL);
+                char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
 
                 if (p) {
                         tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);