18 files changed, 146 insertions, 67 deletions

diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index d37593c2f438..7678f10c4568 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -973,6 +973,29 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
         nr_ioapics++;
 }
 
+int __init acpi_probe_gsi(void)
+{
+        int idx;
+        int gsi;
+        int max_gsi = 0;
+
+        if (acpi_disabled)
+                return 0;
+
+        if (!acpi_ioapic)
+                return 0;
+
+        max_gsi = 0;
+        for (idx = 0; idx < nr_ioapics; idx++) {
+                gsi = mp_ioapic_routing[idx].gsi_end;
+
+                if (gsi > max_gsi)
+                        max_gsi = gsi;
+        }
+
+        return max_gsi + 1;
+}
+
 static void assign_to_mp_irq(struct mp_config_intsrc *m,
                                     struct mp_config_intsrc *mp_irq)
 {
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 707c1f6f95fa..a60c1f3bcb87 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -156,11 +156,11 @@ static int __init acpi_sleep_setup(char *str)
 #ifdef CONFIG_HIBERNATION
                 if (strncmp(str, "s4_nohwsig", 10) == 0)
                         acpi_no_s4_hw_signature();
+                if (strncmp(str, "s4_nonvs", 8) == 0)
+                        acpi_s4_no_nvs();
 #endif
                 if (strncmp(str, "old_ordering", 12) == 0)
                         acpi_old_suspend_ordering();
-                if (strncmp(str, "s4_nonvs", 8) == 0)
-                        acpi_s4_no_nvs();
                 str = strchr(str, ',');
                 if (str != NULL)
                         str += strspn(str, ", \t");
diff --git a/arch/x86/kernel/apic.c b/arch/x86/kernel/apic.c
index 4b6df2469fe3..115449f869ee 100644
--- a/arch/x86/kernel/apic.c
+++ b/arch/x86/kernel/apic.c
@@ -1436,7 +1436,7 @@ static int __init detect_init_APIC(void)
         switch (boot_cpu_data.x86_vendor) {
         case X86_VENDOR_AMD:
                 if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
-                    (boot_cpu_data.x86 == 15))
+                    (boot_cpu_data.x86 >= 15))
                         break;
                 goto no_apic;
         case X86_VENDOR_INTEL:
diff --git a/arch/x86/kernel/cpu/cpufreq/Kconfig b/arch/x86/kernel/cpu/cpufreq/Kconfig
index efae3b22a0ff..65792c2cc462 100644
--- a/arch/x86/kernel/cpu/cpufreq/Kconfig
+++ b/arch/x86/kernel/cpu/cpufreq/Kconfig
@@ -245,17 +245,6 @@ config X86_E_POWERSAVER
 
 comment "shared options"
 
-config X86_ACPI_CPUFREQ_PROC_INTF
-        bool "/proc/acpi/processor/../performance interface (deprecated)"
-        depends on PROC_FS
-        depends on X86_ACPI_CPUFREQ || X86_POWERNOW_K7_ACPI || X86_POWERNOW_K8_ACPI
-        help
-          This enables the deprecated /proc/acpi/processor/../performance
-          interface. While it is helpful for debugging, the generic,
-          cross-architecture cpufreq interfaces should be used.
-
-          If in doubt, say N.
-
 config X86_SPEEDSTEP_LIB
         tristate
         default (X86_SPEEDSTEP_ICH || X86_SPEEDSTEP_SMI || X86_P4_CLOCKMOD)
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
index 5c28b37dea11..fb039cd345d8 100644
--- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
+++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c
@@ -939,10 +939,25 @@ static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data)
         free_cpumask_var(data->acpi_data.shared_cpu_map);
 }
 
+static int get_transition_latency(struct powernow_k8_data *data)
+{
+        int max_latency = 0;
+        int i;
+        for (i = 0; i < data->acpi_data.state_count; i++) {
+                int cur_latency = data->acpi_data.states[i].transition_latency
+                        + data->acpi_data.states[i].bus_master_latency;
+                if (cur_latency > max_latency)
+                        max_latency = cur_latency;
+        }
+        /* value in usecs, needs to be in nanoseconds */
+        return 1000 * max_latency;
+}
+
 #else
 static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) { return -ENODEV; }
 static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) { return; }
 static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index) { return; }
+static int get_transition_latency(struct powernow_k8_data *data) { return 0; }
 #endif /* CONFIG_X86_POWERNOW_K8_ACPI */
 
 /* Take a frequency, and issue the fid/vid transition command */
@@ -1173,7 +1188,13 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
                 if (rc) {
                         goto err_out;
                 }
-        }
+                /* Take a crude guess here.
+                 * That guess was in microseconds, so multiply with 1000 */
+                pol->cpuinfo.transition_latency = (
+                         ((data->rvo + 8) * data->vstable * VST_UNITS_20US) +
+                         ((1 << data->irt) * 30)) * 1000;
+        } else /* ACPI _PSS objects available */
+                pol->cpuinfo.transition_latency = get_transition_latency(data);
 
         /* only run on specific CPU from here on */
         oldmask = current->cpus_allowed;
@@ -1204,11 +1225,6 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
                 cpumask_copy(pol->cpus, &per_cpu(cpu_core_map, pol->cpu));
         data->available_cores = pol->cpus;
 
-        /* Take a crude guess here.
-         * That guess was in microseconds, so multiply with 1000 */
-        pol->cpuinfo.transition_latency = (((data->rvo + 8) * data->vstable * VST_UNITS_20US)
-            + (3 * (1 << data->irt) * 10)) * 1000;
-
         if (cpu_family == CPU_HW_PSTATE)
                 pol->cur = find_khz_freq_from_pstate(data->powernow_table, data->currpstate);
         else
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 430e5c38a544..24ff26a38ade 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -291,6 +291,9 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c)
                 ds_init_intel(c);
         }
 
+        if (c->x86 == 6 && c->x86_model == 29 && cpu_has_clflush)
+                set_cpu_cap(c, X86_FEATURE_CLFLUSH_MONITOR);
+
 #ifdef CONFIG_X86_64
         if (c->x86 == 15)
                 c->x86_cache_alignment = c->x86_clflush_size * 2;
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 48533d77be78..da299eb85fc0 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -36,8 +36,11 @@ static struct _cache_table cache_table[] __cpuinitdata =
 {
         { 0x06, LVL_1_INST, 8 },        /* 4-way set assoc, 32 byte line size */
         { 0x08, LVL_1_INST, 16 },       /* 4-way set assoc, 32 byte line size */
+        { 0x09, LVL_1_INST, 32 },       /* 4-way set assoc, 64 byte line size */
         { 0x0a, LVL_1_DATA, 8 },        /* 2 way set assoc, 32 byte line size */
         { 0x0c, LVL_1_DATA, 16 },       /* 4-way set assoc, 32 byte line size */
+        { 0x0d, LVL_1_DATA, 16 },       /* 4-way set assoc, 64 byte line size */
+        { 0x21, LVL_2,      256 },      /* 8-way set assoc, 64 byte line size */
         { 0x22, LVL_3,      512 },      /* 4-way set assoc, sectored cache, 64 byte line size */
         { 0x23, LVL_3,      1024 },     /* 8-way set assoc, sectored cache, 64 byte line size */
         { 0x25, LVL_3,      2048 },     /* 8-way set assoc, sectored cache, 64 byte line size */
@@ -85,6 +88,18 @@ static struct _cache_table cache_table[] __cpuinitdata =
         { 0x85, LVL_2,    2048 },       /* 8-way set assoc, 32 byte line size */
         { 0x86, LVL_2,     512 },       /* 4-way set assoc, 64 byte line size */
         { 0x87, LVL_2,    1024 },       /* 8-way set assoc, 64 byte line size */
+        { 0xd0, LVL_3,     512 },       /* 4-way set assoc, 64 byte line size */
+        { 0xd1, LVL_3,    1024 },       /* 4-way set assoc, 64 byte line size */
+        { 0xd2, LVL_3,    2048 },       /* 4-way set assoc, 64 byte line size */
+        { 0xd6, LVL_3,    1024 },       /* 8-way set assoc, 64 byte line size */
+        { 0xd7, LVL_3,    2038 },       /* 8-way set assoc, 64 byte line size */
+        { 0xd8, LVL_3,    4096 },       /* 12-way set assoc, 64 byte line size */
+        { 0xdc, LVL_3,    2048 },       /* 12-way set assoc, 64 byte line size */
+        { 0xdd, LVL_3,    4096 },       /* 12-way set assoc, 64 byte line size */
+        { 0xde, LVL_3,    8192 },       /* 12-way set assoc, 64 byte line size */
+        { 0xe2, LVL_3,    2048 },       /* 16-way set assoc, 64 byte line size */
+        { 0xe3, LVL_3,    4096 },       /* 16-way set assoc, 64 byte line size */
+        { 0xe4, LVL_3,    8192 },       /* 16-way set assoc, 64 byte line size */
         { 0x00, 0, 0}
 };
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e28c7a987793..a1346217e43c 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -346,6 +346,7 @@ ENTRY(save_args)
         popq_cfi %rax                   /* move return address... */
         mov %gs:pda_irqstackptr,%rsp
         EMPTY_FRAME 0
+        pushq_cfi %rbp                  /* backlink for unwinder */
         pushq_cfi %rax                  /* ... to the new stack */
         /*
          * We entered an interrupt context - irqs are off:
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 1b43086b097a..231bdd3c5b1c 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -488,20 +488,21 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
          * ignore such a protection.
          */
         asm volatile(
-                "1: " _ASM_MOV " (%[parent_old]), %[old]\n"
-                "2: " _ASM_MOV " %[return_hooker], (%[parent_replaced])\n"
+                "1: " _ASM_MOV " (%[parent]), %[old]\n"
+                "2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
                 "   movl $0, %[faulted]\n"
+                "3:\n"
 
                 ".section .fixup, \"ax\"\n"
-                "3: movl $1, %[faulted]\n"
+                "4: movl $1, %[faulted]\n"
+                "   jmp 3b\n"
                 ".previous\n"
 
-                _ASM_EXTABLE(1b, 3b)
-                _ASM_EXTABLE(2b, 3b)
+                _ASM_EXTABLE(1b, 4b)
+                _ASM_EXTABLE(2b, 4b)
 
-                : [parent_replaced] "=r" (parent), [old] "=r" (old),
-                  [faulted] "=r" (faulted)
-                : [parent_old] "0" (parent), [return_hooker] "r" (return_hooker)
+                : [old] "=r" (old), [faulted] "=r" (faulted)
+                : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
                 : "memory"
         );
 
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index 64d5ad0b8add..388254f69a2a 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -897,7 +897,7 @@ static unsigned long hpet_rtc_flags;
 static int hpet_prev_update_sec;
 static struct rtc_time hpet_alarm_time;
 static unsigned long hpet_pie_count;
-static unsigned long hpet_t1_cmp;
+static u32 hpet_t1_cmp;
 static unsigned long hpet_default_delta;
 static unsigned long hpet_pie_delta;
 static unsigned long hpet_pie_limit;
@@ -905,6 +905,14 @@ static unsigned long hpet_pie_limit;
 static rtc_irq_handler irq_handler;
 
 /*
+ * Check that the hpet counter c1 is ahead of the c2
+ */
+static inline int hpet_cnt_ahead(u32 c1, u32 c2)
+{
+        return (s32)(c2 - c1) < 0;
+}
+
+/*
  * Registers a IRQ handler.
  */
 int hpet_register_irq_handler(rtc_irq_handler handler)
@@ -1075,7 +1083,7 @@ static void hpet_rtc_timer_reinit(void)
                 hpet_t1_cmp += delta;
                 hpet_writel(hpet_t1_cmp, HPET_T1_CMP);
                 lost_ints++;
-        } while ((long)(hpet_readl(HPET_COUNTER) - hpet_t1_cmp) > 0);
+        } while (!hpet_cnt_ahead(hpet_t1_cmp, hpet_readl(HPET_COUNTER)));
 
         if (lost_ints) {
                 if (hpet_rtc_flags & RTC_PIE)
diff --git a/arch/x86/kernel/i8237.c b/arch/x86/kernel/i8237.c
index dbd6c1d1b638..b42ca694dc68 100644
--- a/arch/x86/kernel/i8237.c
+++ b/arch/x86/kernel/i8237.c
@@ -28,10 +28,10 @@ static int i8237A_resume(struct sys_device *dev)
 
         flags = claim_dma_lock();
 
-        dma_outb(DMA1_RESET_REG, 0);
-        dma_outb(DMA2_RESET_REG, 0);
+        dma_outb(0, DMA1_RESET_REG);
+        dma_outb(0, DMA2_RESET_REG);
 
-        for (i = 0;i < 8;i++) {
+        for (i = 0; i < 8; i++) {
                 set_dma_addr(i, 0x000000);
                 /* DMA count is a bit weird so this is not 0 */
                 set_dma_count(i, 1);
@@ -51,14 +51,14 @@ static int i8237A_suspend(struct sys_device *dev, pm_message_t state)
 }
 
 static struct sysdev_class i8237_sysdev_class = {
-        .name = "i8237",
-        .suspend = i8237A_suspend,
-        .resume = i8237A_resume,
+        .name           = "i8237",
+        .suspend        = i8237A_suspend,
+        .resume         = i8237A_resume,
 };
 
 static struct sys_device device_i8237A = {
-        .id     = 0,
-        .cls    = &i8237_sysdev_class,
+        .id             = 0,
+        .cls            = &i8237_sysdev_class,
 };
 
 static int __init i8237A_init_sysfs(void)
@@ -68,5 +68,4 @@ static int __init i8237A_init_sysfs(void)
                 error = sysdev_register(&device_i8237A);
         return error;
 }
-
 device_initcall(i8237A_init_sysfs);
diff --git a/arch/x86/kernel/io_apic.c b/arch/x86/kernel/io_apic.c
index 1c4a1302536c..bc7ac4da90d7 100644
--- a/arch/x86/kernel/io_apic.c
+++ b/arch/x86/kernel/io_apic.c
@@ -2528,14 +2528,15 @@ static void irq_complete_move(struct irq_desc **descp)
 
         vector = ~get_irq_regs()->orig_ax;
         me = smp_processor_id();
+
+        if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain)) {
 #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
                 *descp = desc = move_irq_desc(desc, me);
                 /* get the new one */
                 cfg = desc->chip_data;
 #endif
-
-        if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
                 send_cleanup_vector(cfg);
+        }
 }
 #else
 static inline void irq_complete_move(struct irq_desc **descp) {}
@@ -3840,14 +3841,24 @@ int __init io_apic_get_redir_entries (int ioapic)
 
 void __init probe_nr_irqs_gsi(void)
 {
-        int idx;
         int nr = 0;
 
-        for (idx = 0; idx < nr_ioapics; idx++)
-                nr += io_apic_get_redir_entries(idx) + 1;
-
-        if (nr > nr_irqs_gsi)
+        nr = acpi_probe_gsi();
+        if (nr > nr_irqs_gsi) {
                 nr_irqs_gsi = nr;
+        } else {
+                /* for acpi=off or acpi is not compiled in */
+                int idx;
+
+                nr = 0;
+                for (idx = 0; idx < nr_ioapics; idx++)
+                        nr += io_apic_get_redir_entries(idx) + 1;
+
+                if (nr > nr_irqs_gsi)
+                        nr_irqs_gsi = nr;
+        }
+
+        printk(KERN_DEBUG "nr_irqs_gsi: %d\n", nr_irqs_gsi);
 }
 
 /* --------------------------------------------------------------------------
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c
index 1507ad4e674d..10a09c2f1828 100644
--- a/arch/x86/kernel/irqinit_32.c
+++ b/arch/x86/kernel/irqinit_32.c
@@ -78,15 +78,6 @@ void __init init_ISA_irqs(void)
         }
 }
 
-/*
- * IRQ2 is cascade interrupt to second interrupt controller
- */
-static struct irqaction irq2 = {
-        .handler = no_action,
-        .mask = CPU_MASK_NONE,
-        .name = "cascade",
-};
-
 DEFINE_PER_CPU(vector_irq_t, vector_irq) = {
         [0 ... IRQ0_VECTOR - 1] = -1,
         [IRQ0_VECTOR] = 0,
@@ -178,9 +169,6 @@ void __init native_init_IRQ(void)
         alloc_intr_gate(THERMAL_APIC_VECTOR, thermal_interrupt);
 #endif
 
-        if (!acpi_ioapic)
-                setup_irq(2, &irq2);
-
         /* setup after call gates are initialised (usually add in
          * the architecture specific gates)
          */
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index e68bb9e30864..6d12f7e37f8c 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -180,6 +180,9 @@ void mwait_idle_with_hints(unsigned long ax, unsigned long cx)
 
         trace_power_start(&it, POWER_CSTATE, (ax>>4)+1);
         if (!need_resched()) {
+                if (cpu_has(&current_cpu_data, X86_FEATURE_CLFLUSH_MONITOR))
+                        clflush((void *)&current_thread_info()->flags);
+
                 __monitor((void *)&current_thread_info()->flags, 0, 0);
                 smp_mb();
                 if (!need_resched())
@@ -194,6 +197,9 @@ static void mwait_idle(void)
         struct power_trace it;
         if (!need_resched()) {
                 trace_power_start(&it, POWER_CSTATE, 1);
+                if (cpu_has(&current_cpu_data, X86_FEATURE_CLFLUSH_MONITOR))
+                        clflush((void *)&current_thread_info()->flags);
+
                 __monitor((void *)&current_thread_info()->flags, 0, 0);
                 smp_mb();
                 if (!need_resched())
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 416fb9282f4f..85b4cb5c1980 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -40,6 +40,7 @@
 #include <linux/uaccess.h>
 #include <linux/io.h>
 #include <linux/ftrace.h>
+#include <linux/dmi.h>
 
 #include <asm/pgtable.h>
 #include <asm/system.h>
@@ -151,14 +152,18 @@ void __show_regs(struct pt_regs *regs, int all)
         unsigned long d0, d1, d2, d3, d6, d7;
         unsigned int fsindex, gsindex;
         unsigned int ds, cs, es;
+        const char *board;
 
         printk("\n");
         print_modules();
-        printk(KERN_INFO "Pid: %d, comm: %.20s %s %s %.*s\n",
+        board = dmi_get_system_info(DMI_PRODUCT_NAME);
+        if (!board)
+                board = "";
+        printk(KERN_INFO "Pid: %d, comm: %.20s %s %s %.*s %s\n",
                 current->pid, current->comm, print_tainted(),
                 init_utsname()->release,
                 (int)strcspn(init_utsname()->version, " "),
-                init_utsname()->version);
+                init_utsname()->version, board);
         printk(KERN_INFO "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
         printk_address(regs->ip, 1);
         printk(KERN_INFO "RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss,
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index ae0d8042cf69..c461f6d69074 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -607,7 +607,7 @@ struct x86_quirks *x86_quirks __initdata = &default_x86_quirks;
 static int __init dmi_low_memory_corruption(const struct dmi_system_id *d)
 {
         printk(KERN_NOTICE
-                "%s detected: BIOS may corrupt low RAM, working it around.\n",
+                "%s detected: BIOS may corrupt low RAM, working around it.\n",
                 d->ident);
 
         e820_update_range(0, 0x10000, E820_RAM, E820_RESERVED);
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 98c2d055284b..7932338d7cb3 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -896,7 +896,7 @@ asmlinkage void math_state_restore(void)
 EXPORT_SYMBOL_GPL(math_state_restore);
 
 #ifndef CONFIG_MATH_EMULATION
-asmlinkage void math_emulate(long arg)
+void math_emulate(struct math_emu_info *info)
 {
         printk(KERN_EMERG
                 "math-emulation not enabled and no coprocessor found.\n");
@@ -906,16 +906,19 @@ asmlinkage void math_emulate(long arg)
 }
 #endif /* CONFIG_MATH_EMULATION */
 
-dotraplinkage void __kprobes
-do_device_not_available(struct pt_regs *regs, long error)
+dotraplinkage void __kprobes do_device_not_available(struct pt_regs regs)
 {
 #ifdef CONFIG_X86_32
         if (read_cr0() & X86_CR0_EM) {
-                conditional_sti(regs);
-                math_emulate(0);
+                struct math_emu_info info = { };
+
+                conditional_sti(&regs);
+
+                info.regs = &regs;
+                math_emulate(&info);
         } else {
                 math_state_restore(); /* interrupts still off */
-                conditional_sti(regs);
+                conditional_sti(&regs);
         }
 #else
         math_state_restore();
diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c
index 1d3302cc2ddf..bef58b4982db 100644
--- a/arch/x86/kernel/vmi_32.c
+++ b/arch/x86/kernel/vmi_32.c
@@ -321,6 +321,16 @@ static void vmi_release_pmd(unsigned long pfn)
 }
 
 /*
+ * We use the pgd_free hook for releasing the pgd page:
+ */
+static void vmi_pgd_free(struct mm_struct *mm, pgd_t *pgd)
+{
+        unsigned long pfn = __pa(pgd) >> PAGE_SHIFT;
+
+        vmi_ops.release_page(pfn, VMI_PAGE_L2);
+}
+
+/*
  * Helper macros for MMU update flags.  We can defer updates until a flush
  * or page invalidation only if the update is to the current address space
  * (otherwise, there is no flush).  We must check against init_mm, since
@@ -762,6 +772,7 @@ static inline int __init activate_vmi(void)
         if (vmi_ops.release_page) {
                 pv_mmu_ops.release_pte = vmi_release_pte;
                 pv_mmu_ops.release_pmd = vmi_release_pmd;
+                pv_mmu_ops.pgd_free = vmi_pgd_free;
         }
 
         /* Set linear is needed in all cases */