86 files changed, 2159 insertions, 1680 deletions
diff --git a/Documentation/cpu-hotplug.txt b/Documentation/cpu-hotplug.txt
index a0b005d2bd95..f9ad5e048b11 100644
--- a/Documentation/cpu-hotplug.txt
+++ b/Documentation/cpu-hotplug.txt
@@ -108,7 +108,7 @@ Never use anything other than cpumask_t to represent bitmap of CPUs.
        for_each_possible_cpu     - Iterate over cpu_possible_mask
        for_each_online_cpu       - Iterate over cpu_online_mask
        for_each_present_cpu      - Iterate over cpu_present_mask
-        for_each_cpu_mask(x,mask) - Iterate over some random collection of cpu mask.
+        for_each_cpu(x,mask)      - Iterate over some random collection of cpu mask.
        #include <linux/cpu.h>
        get_online_cpus() and put_online_cpus():
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index ffe8e1b814e0..714411f62391 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -636,7 +636,7 @@ void smp_send_stop(void)
                cpumask_t mask;
                cpumask_copy(&mask, cpu_online_mask);
-                cpu_clear(smp_processor_id(), mask);
+                cpumask_clear_cpu(smp_processor_id(), &mask);
                smp_cross_call(&mask, IPI_CPU_STOP);
        }
diff --git a/arch/blackfin/mach-bf561/smp.c b/arch/blackfin/mach-bf561/smp.c
index 11789beca75a..8c0c80fd1a45 100644
--- a/arch/blackfin/mach-bf561/smp.c
+++ b/arch/blackfin/mach-bf561/smp.c
@@ -124,7 +124,7 @@ void platform_send_ipi(cpumask_t callmap, int irq)
        unsigned int cpu;
        int offset = (irq == IRQ_SUPPLE_0) ? 6 : 8;
-        for_each_cpu_mask(cpu, callmap) {
+        for_each_cpu(cpu, &callmap) {
                BUG_ON(cpu >= 2);
                SSYNC();
                bfin_write_SICB_SYSCR(bfin_read_SICB_SYSCR() | (1 << (offset + cpu)));
diff --git a/arch/ia64/include/asm/acpi.h b/arch/ia64/include/asm/acpi.h
index a1d91ab4c5ef..aa0fdf125aba 100644
--- a/arch/ia64/include/asm/acpi.h
+++ b/arch/ia64/include/asm/acpi.h
@@ -117,7 +117,7 @@ static inline void arch_acpi_set_pdc_bits(u32 *buf)
 #ifdef CONFIG_ACPI_NUMA
 extern cpumask_t early_cpu_possible_map;
 #define for_each_possible_early_cpu(cpu)  \
-        for_each_cpu_mask((cpu), early_cpu_possible_map)
+        for_each_cpu((cpu), &early_cpu_possible_map)
 static inline void per_cpu_scan_finalize(int min_cpus, int reserve_cpus)
 {
@@ -125,13 +125,13 @@ static inline void per_cpu_scan_finalize(int min_cpus, int reserve_cpus)
        int cpu;
        int next_nid = 0;
-        low_cpu = cpus_weight(early_cpu_possible_map);
+        low_cpu = cpumask_weight(&early_cpu_possible_map);
        high_cpu = max(low_cpu, min_cpus);
        high_cpu = min(high_cpu + reserve_cpus, NR_CPUS);
        for (cpu = low_cpu; cpu < high_cpu; cpu++) {
-                cpu_set(cpu, early_cpu_possible_map);
+                cpumask_set_cpu(cpu, &early_cpu_possible_map);
                if (node_cpuid[cpu].nid == NUMA_NO_NODE) {
                        node_cpuid[cpu].nid = next_nid;
                        next_nid++;
diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c
index 2c4498919d3c..35bf22cc71b7 100644
--- a/arch/ia64/kernel/acpi.c
+++ b/arch/ia64/kernel/acpi.c
@@ -483,7 +483,7 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
            (pa->apic_id << 8) | (pa->local_sapic_eid);
        /* nid should be overridden as logical node id later */
        node_cpuid[srat_num_cpus].nid = pxm;
-        cpu_set(srat_num_cpus, early_cpu_possible_map);
+        cpumask_set_cpu(srat_num_cpus, &early_cpu_possible_map);
        srat_num_cpus++;
 }
diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c
index cd44a57c73be..bc9501e36e77 100644
--- a/arch/ia64/kernel/iosapic.c
+++ b/arch/ia64/kernel/iosapic.c
@@ -690,7 +690,7 @@ skip_numa_setup:
        do {
                if (++cpu >= nr_cpu_ids)
                        cpu = 0;
-        } while (!cpu_online(cpu) || !cpu_isset(cpu, domain));
+        } while (!cpu_online(cpu) || !cpumask_test_cpu(cpu, &domain));
        return cpu_physical_id(cpu);
 #else  /* CONFIG_SMP */
diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c
index 698d8fefde6c..eaa3199f98c8 100644
--- a/arch/ia64/kernel/irq_ia64.c
+++ b/arch/ia64/kernel/irq_ia64.c
@@ -109,13 +109,13 @@ static inline int find_unassigned_vector(cpumask_t domain)
        int pos, vector;
        cpumask_and(&mask, &domain, cpu_online_mask);
-        if (cpus_empty(mask))
+        if (cpumask_empty(&mask))
                return -EINVAL;
        for (pos = 0; pos < IA64_NUM_DEVICE_VECTORS; pos++) {
                vector = IA64_FIRST_DEVICE_VECTOR + pos;
-                cpus_and(mask, domain, vector_table[vector]);
+                cpumask_and(&mask, &domain, &vector_table[vector]);
-                if (!cpus_empty(mask))
+                if (!cpumask_empty(&mask))
                        continue;
                return vector;
        }
@@ -132,18 +132,18 @@ static int __bind_irq_vector(int irq, int vector, cpumask_t domain)
        BUG_ON((unsigned)vector >= IA64_NUM_VECTORS);
        cpumask_and(&mask, &domain, cpu_online_mask);
-        if (cpus_empty(mask))
+        if (cpumask_empty(&mask))
                return -EINVAL;
-        if ((cfg->vector == vector) && cpus_equal(cfg->domain, domain))
+        if ((cfg->vector == vector) && cpumask_equal(&cfg->domain, &domain))
                return 0;
        if (cfg->vector != IRQ_VECTOR_UNASSIGNED)
                return -EBUSY;
-        for_each_cpu_mask(cpu, mask)
+        for_each_cpu(cpu, &mask)
                per_cpu(vector_irq, cpu)[vector] = irq;
        cfg->vector = vector;
        cfg->domain = domain;
        irq_status[irq] = IRQ_USED;
-        cpus_or(vector_table[vector], vector_table[vector], domain);
+        cpumask_or(&vector_table[vector], &vector_table[vector], &domain);
        return 0;
 }
@@ -161,7 +161,6 @@ int bind_irq_vector(int irq, int vector, cpumask_t domain)
 static void __clear_irq_vector(int irq)
 {
        int vector, cpu;
-        cpumask_t mask;
        cpumask_t domain;
        struct irq_cfg *cfg = &irq_cfg[irq];
@@ -169,13 +168,12 @@ static void __clear_irq_vector(int irq)
        BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED);
        vector = cfg->vector;
        domain = cfg->domain;
-        cpumask_and(&mask, &cfg->domain, cpu_online_mask);
+        for_each_cpu_and(cpu, &cfg->domain, cpu_online_mask)
-        for_each_cpu_mask(cpu, mask)
                per_cpu(vector_irq, cpu)[vector] = -1;
        cfg->vector = IRQ_VECTOR_UNASSIGNED;
        cfg->domain = CPU_MASK_NONE;
        irq_status[irq] = IRQ_UNUSED;
-        cpus_andnot(vector_table[vector], vector_table[vector], domain);
+        cpumask_andnot(&vector_table[vector], &vector_table[vector], &domain);
 }
 static void clear_irq_vector(int irq)
@@ -244,7 +242,7 @@ void __setup_vector_irq(int cpu)
                per_cpu(vector_irq, cpu)[vector] = -1;
        /* Mark the inuse vectors */
        for (irq = 0; irq < NR_IRQS; ++irq) {
-                if (!cpu_isset(cpu, irq_cfg[irq].domain))
+                if (!cpumask_test_cpu(cpu, &irq_cfg[irq].domain))
                        continue;
                vector = irq_to_vector(irq);
                per_cpu(vector_irq, cpu)[vector] = irq;
@@ -261,7 +259,7 @@ static enum vector_domain_type {
 static cpumask_t vector_allocation_domain(int cpu)
 {
        if (vector_domain_type == VECTOR_DOMAIN_PERCPU)
-                return cpumask_of_cpu(cpu);
+                return *cpumask_of(cpu);
        return CPU_MASK_ALL;
 }
@@ -275,7 +273,7 @@ static int __irq_prepare_move(int irq, int cpu)
                return -EBUSY;
        if (cfg->vector == IRQ_VECTOR_UNASSIGNED || !cpu_online(cpu))
                return -EINVAL;
-        if (cpu_isset(cpu, cfg->domain))
+        if (cpumask_test_cpu(cpu, &cfg->domain))
                return 0;
        domain = vector_allocation_domain(cpu);
        vector = find_unassigned_vector(domain);
@@ -309,12 +307,12 @@ void irq_complete_move(unsigned irq)
        if (likely(!cfg->move_in_progress))
                return;
-        if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain)))
+        if (unlikely(cpumask_test_cpu(smp_processor_id(), &cfg->old_domain)))
                return;
        cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask);
-        cfg->move_cleanup_count = cpus_weight(cleanup_mask);
+        cfg->move_cleanup_count = cpumask_weight(&cleanup_mask);
-        for_each_cpu_mask(i, cleanup_mask)
+        for_each_cpu(i, &cleanup_mask)
                platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0);
        cfg->move_in_progress = 0;
 }
@@ -340,12 +338,12 @@ static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id)
                if (!cfg->move_cleanup_count)
                        goto unlock;
-                if (!cpu_isset(me, cfg->old_domain))
+                if (!cpumask_test_cpu(me, &cfg->old_domain))
                        goto unlock;
                spin_lock_irqsave(&vector_lock, flags);
                __this_cpu_write(vector_irq[vector], -1);
-                cpu_clear(me, vector_table[vector]);
+                cpumask_clear_cpu(me, &vector_table[vector]);
                spin_unlock_irqrestore(&vector_lock, flags);
                cfg->move_cleanup_count--;
        unlock:
diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c
index 8bfd36af46f8..dd5801eb4c69 100644
--- a/arch/ia64/kernel/mca.c
+++ b/arch/ia64/kernel/mca.c
@@ -1293,7 +1293,7 @@ ia64_mca_handler(struct pt_regs *regs, struct switch_stack *sw,
                monarch_cpu = cpu;
                sos->monarch = 1;
        } else {
-                cpu_set(cpu, mca_cpu);
+                cpumask_set_cpu(cpu, &mca_cpu);
                sos->monarch = 0;
        }
        mprintk(KERN_INFO "Entered OS MCA handler. PSP=%lx cpu=%d "
@@ -1316,7 +1316,7 @@ ia64_mca_handler(struct pt_regs *regs, struct switch_stack *sw,
                 */
                ia64_mca_wakeup_all();
        } else {
-                while (cpu_isset(cpu, mca_cpu))
+                while (cpumask_test_cpu(cpu, &mca_cpu))
                        cpu_relax();    /* spin until monarch wakes us */
        }
@@ -1355,9 +1355,9 @@ ia64_mca_handler(struct pt_regs *regs, struct switch_stack *sw,
                 * and put this cpu in the rendez loop.
                 */
                for_each_online_cpu(i) {
-                        if (cpu_isset(i, mca_cpu)) {
+                        if (cpumask_test_cpu(i, &mca_cpu)) {
                                monarch_cpu = i;
-                                cpu_clear(i, mca_cpu);  /* wake next cpu */
+                                cpumask_clear_cpu(i, &mca_cpu); /* wake next cpu */
                                while (monarch_cpu != -1)
                                        cpu_relax();    /* spin until last cpu leaves */
                                set_curr_task(cpu, previous_current);
@@ -1822,7 +1822,7 @@ format_mca_init_stack(void *mca_data, unsigned long offset,
        ti->cpu = cpu;
        p->stack = ti;
        p->state = TASK_UNINTERRUPTIBLE;
-        cpu_set(cpu, p->cpus_allowed);
+        cpumask_set_cpu(cpu, &p->cpus_allowed);
        INIT_LIST_HEAD(&p->tasks);
        p->parent = p->real_parent = p->group_leader = p;
        INIT_LIST_HEAD(&p->children);
diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c
index 8ae36ea177d3..9dd7464f8c17 100644
--- a/arch/ia64/kernel/msi_ia64.c
+++ b/arch/ia64/kernel/msi_ia64.c
@@ -47,15 +47,14 @@ int ia64_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc)
        struct msi_msg  msg;
        unsigned long   dest_phys_id;
        int     irq, vector;
-        cpumask_t mask;
        irq = create_irq();
        if (irq < 0)
                return irq;
        irq_set_msi_desc(irq, desc);
-        cpumask_and(&mask, &(irq_to_domain(irq)), cpu_online_mask);
+        dest_phys_id = cpu_physical_id(cpumask_any_and(&(irq_to_domain(irq)),
-        dest_phys_id = cpu_physical_id(first_cpu(mask));
+                                                       cpu_online_mask));
        vector = irq_to_vector(irq);
        msg.address_hi = 0;
@@ -171,10 +170,9 @@ msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
 {
        struct irq_cfg *cfg = irq_cfg + irq;
        unsigned dest;
-        cpumask_t mask;
-        cpumask_and(&mask, &(irq_to_domain(irq)), cpu_online_mask);
+        dest = cpu_physical_id(cpumask_first_and(&(irq_to_domain(irq)),
-        dest = cpu_physical_id(first_cpu(mask));
+                                                 cpu_online_mask));
        msg->address_hi = 0;
        msg->address_lo =
diff --git a/arch/ia64/kernel/numa.c b/arch/ia64/kernel/numa.c
index d288cde93606..92c376279c6d 100644
--- a/arch/ia64/kernel/numa.c
+++ b/arch/ia64/kernel/numa.c
@@ -39,7 +39,7 @@ void map_cpu_to_node(int cpu, int nid)
        }
        /* sanity check first */
        oldnid = cpu_to_node_map[cpu];
-        if (cpu_isset(cpu, node_to_cpu_mask[oldnid])) {
+        if (cpumask_test_cpu(cpu, &node_to_cpu_mask[oldnid])) {
                return; /* nothing to do */
        }
        /* we don't have cpu-driven node hot add yet...
@@ -47,16 +47,16 @@ void map_cpu_to_node(int cpu, int nid)
        if (!node_online(nid))
                nid = first_online_node;
        cpu_to_node_map[cpu] = nid;
-        cpu_set(cpu, node_to_cpu_mask[nid]);
+        cpumask_set_cpu(cpu, &node_to_cpu_mask[nid]);
        return;
 }
 void unmap_cpu_from_node(int cpu, int nid)
 {
-        WARN_ON(!cpu_isset(cpu, node_to_cpu_mask[nid]));
+        WARN_ON(!cpumask_test_cpu(cpu, &node_to_cpu_mask[nid]));
        WARN_ON(cpu_to_node_map[cpu] != nid);
        cpu_to_node_map[cpu] = 0;
-        cpu_clear(cpu, node_to_cpu_mask[nid]);
+        cpumask_clear_cpu(cpu, &node_to_cpu_mask[nid]);
 }
@@ -71,7 +71,7 @@ void __init build_cpu_to_node_map(void)
        int cpu, i, node;
        for(node=0; node < MAX_NUMNODES; node++)
-                cpus_clear(node_to_cpu_mask[node]);
+                cpumask_clear(&node_to_cpu_mask[node]);
        for_each_possible_early_cpu(cpu) {
                node = -1;
diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c
index ee9719eebb1e..1eeffb7fbb16 100644
--- a/arch/ia64/kernel/salinfo.c
+++ b/arch/ia64/kernel/salinfo.c
@@ -256,7 +256,7 @@ salinfo_log_wakeup(int type, u8 *buffer, u64 size, int irqsafe)
                        data_saved->buffer = buffer;
                }
        }
-        cpu_set(smp_processor_id(), data->cpu_event);
+        cpumask_set_cpu(smp_processor_id(), &data->cpu_event);
        if (irqsafe) {
                salinfo_work_to_do(data);
                spin_unlock_irqrestore(&data_saved_lock, flags);
@@ -274,7 +274,7 @@ salinfo_timeout_check(struct salinfo_data *data)
        unsigned long flags;
        if (!data->open)
                return;
-        if (!cpus_empty(data->cpu_event)) {
+        if (!cpumask_empty(&data->cpu_event)) {
                spin_lock_irqsave(&data_saved_lock, flags);
                salinfo_work_to_do(data);
                spin_unlock_irqrestore(&data_saved_lock, flags);
@@ -308,7 +308,7 @@ salinfo_event_read(struct file *file, char __user *buffer, size_t count, loff_t
        int i, n, cpu = -1;
 retry:
-        if (cpus_empty(data->cpu_event) && down_trylock(&data->mutex)) {
+        if (cpumask_empty(&data->cpu_event) && down_trylock(&data->mutex)) {
                if (file->f_flags & O_NONBLOCK)
                        return -EAGAIN;
                if (down_interruptible(&data->mutex))
@@ -317,9 +317,9 @@ retry:
        n = data->cpu_check;
        for (i = 0; i < nr_cpu_ids; i++) {
-                if (cpu_isset(n, data->cpu_event)) {
+                if (cpumask_test_cpu(n, &data->cpu_event)) {
                        if (!cpu_online(n)) {
-                                cpu_clear(n, data->cpu_event);
+                                cpumask_clear_cpu(n, &data->cpu_event);
                                continue;
                        }
                        cpu = n;
@@ -451,7 +451,7 @@ retry:
                call_on_cpu(cpu, salinfo_log_read_cpu, data);
        if (!data->log_size) {
                data->state = STATE_NO_DATA;
-                cpu_clear(cpu, data->cpu_event);
+                cpumask_clear_cpu(cpu, &data->cpu_event);
        } else {
                data->state = STATE_LOG_RECORD;
        }
@@ -491,11 +491,11 @@ salinfo_log_clear(struct salinfo_data *data, int cpu)
        unsigned long flags;
        spin_lock_irqsave(&data_saved_lock, flags);
        data->state = STATE_NO_DATA;
-        if (!cpu_isset(cpu, data->cpu_event)) {
+        if (!cpumask_test_cpu(cpu, &data->cpu_event)) {
                spin_unlock_irqrestore(&data_saved_lock, flags);
                return 0;
        }
-        cpu_clear(cpu, data->cpu_event);
+        cpumask_clear_cpu(cpu, &data->cpu_event);
        if (data->saved_num) {
                shift1_data_saved(data, data->saved_num - 1);
                data->saved_num = 0;
@@ -509,7 +509,7 @@ salinfo_log_clear(struct salinfo_data *data, int cpu)
        salinfo_log_new_read(cpu, data);
        if (data->state == STATE_LOG_RECORD) {
                spin_lock_irqsave(&data_saved_lock, flags);
-                cpu_set(cpu, data->cpu_event);
+                cpumask_set_cpu(cpu, &data->cpu_event);
                salinfo_work_to_do(data);
                spin_unlock_irqrestore(&data_saved_lock, flags);
        }
@@ -581,7 +581,7 @@ salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu
                for (i = 0, data = salinfo_data;
                     i < ARRAY_SIZE(salinfo_data);
                     ++i, ++data) {
-                        cpu_set(cpu, data->cpu_event);
+                        cpumask_set_cpu(cpu, &data->cpu_event);
                        salinfo_work_to_do(data);
                }
                spin_unlock_irqrestore(&data_saved_lock, flags);
@@ -601,7 +601,7 @@ salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu
                                        shift1_data_saved(data, j);
                                }
                        }
-                        cpu_clear(cpu, data->cpu_event);
+                        cpumask_clear_cpu(cpu, &data->cpu_event);
                }
                spin_unlock_irqrestore(&data_saved_lock, flags);
                break;
@@ -659,7 +659,7 @@ salinfo_init(void)
                /* we missed any events before now */
                for_each_online_cpu(j)
-                        cpu_set(j, data->cpu_event);
+                        cpumask_set_cpu(j, &data->cpu_event);
                *sdir++ = dir;
        }
diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c
index d86669bcdfb2..b9761389cb8d 100644
--- a/arch/ia64/kernel/setup.c
+++ b/arch/ia64/kernel/setup.c
@@ -562,8 +562,8 @@ setup_arch (char **cmdline_p)
 #  ifdef CONFIG_ACPI_HOTPLUG_CPU
        prefill_possible_map();
 #  endif
-        per_cpu_scan_finalize((cpus_weight(early_cpu_possible_map) == 0 ?
+        per_cpu_scan_finalize((cpumask_weight(&early_cpu_possible_map) == 0 ?
-                32 : cpus_weight(early_cpu_possible_map)),
+                32 : cpumask_weight(&early_cpu_possible_map)),
                additional_cpus > 0 ? additional_cpus : 0);
 # endif
 #endif /* CONFIG_APCI_BOOT */
@@ -702,7 +702,8 @@ show_cpuinfo (struct seq_file *m, void *v)
                   c->itc_freq / 1000000, c->itc_freq % 1000000,
                   lpj*HZ/500000, (lpj*HZ/5000) % 100);
 #ifdef CONFIG_SMP
-        seq_printf(m, "siblings   : %u\n", cpus_weight(cpu_core_map[cpunum]));
+        seq_printf(m, "siblings   : %u\n",
+                   cpumask_weight(&cpu_core_map[cpunum]));
        if (c->socket_id != -1)
                seq_printf(m, "physical id: %u\n", c->socket_id);
        if (c->threads_per_core > 1 || c->cores_per_socket > 1)
@@ -933,8 +934,8 @@ cpu_init (void)
         * (must be done after per_cpu area is setup)
         */
        if (smp_processor_id() == 0) {
-                cpu_set(0, per_cpu(cpu_sibling_map, 0));
+                cpumask_set_cpu(0, &per_cpu(cpu_sibling_map, 0));
-                cpu_set(0, cpu_core_map[0]);
+                cpumask_set_cpu(0, &cpu_core_map[0]);
        } else {
                /*
                 * Set ar.k3 so that assembly code in MCA handler can compute
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 9fcd4e63048f..7f706d4f84f7 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -262,11 +262,11 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
        preempt_disable();
        mycpu = smp_processor_id();
-        for_each_cpu_mask(cpu, cpumask)
+        for_each_cpu(cpu, &cpumask)
                counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff;
        mb();
-        for_each_cpu_mask(cpu, cpumask) {
+        for_each_cpu(cpu, &cpumask) {
                if (cpu == mycpu)
                        flush_mycpu = 1;
                else
@@ -276,7 +276,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
        if (flush_mycpu)
                smp_local_flush_tlb();
-        for_each_cpu_mask(cpu, cpumask)
+        for_each_cpu(cpu, &cpumask)
                while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff))
                        udelay(FLUSH_DELAY);
diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c
index 547a48d78bd7..15051e9c2c6f 100644
--- a/arch/ia64/kernel/smpboot.c
+++ b/arch/ia64/kernel/smpboot.c
@@ -434,7 +434,7 @@ smp_callin (void)
        /*
         * Allow the master to continue.
         */
-        cpu_set(cpuid, cpu_callin_map);
+        cpumask_set_cpu(cpuid, &cpu_callin_map);
        Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid);
 }
@@ -475,13 +475,13 @@ do_boot_cpu (int sapicid, int cpu, struct task_struct *idle)
         */
        Dprintk("Waiting on callin_map ...");
        for (timeout = 0; timeout < 100000; timeout++) {
-                if (cpu_isset(cpu, cpu_callin_map))
+                if (cpumask_test_cpu(cpu, &cpu_callin_map))
                        break;  /* It has booted */
                udelay(100);
        }
        Dprintk("\n");
-        if (!cpu_isset(cpu, cpu_callin_map)) {
+        if (!cpumask_test_cpu(cpu, &cpu_callin_map)) {
                printk(KERN_ERR "Processor 0x%x/0x%x is stuck.\n", cpu, sapicid);
                ia64_cpu_to_sapicid[cpu] = -1;
                set_cpu_online(cpu, false);  /* was set in smp_callin() */
@@ -541,7 +541,7 @@ smp_prepare_cpus (unsigned int max_cpus)
        smp_setup_percpu_timer();
-        cpu_set(0, cpu_callin_map);
+        cpumask_set_cpu(0, &cpu_callin_map);
        local_cpu_data->loops_per_jiffy = loops_per_jiffy;
        ia64_cpu_to_sapicid[0] = boot_cpu_id;
@@ -565,7 +565,7 @@ smp_prepare_cpus (unsigned int max_cpus)
 void smp_prepare_boot_cpu(void)
 {
        set_cpu_online(smp_processor_id(), true);
-        cpu_set(smp_processor_id(), cpu_callin_map);
+        cpumask_set_cpu(smp_processor_id(), &cpu_callin_map);
        set_numa_node(cpu_to_node_map[smp_processor_id()]);
        per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
        paravirt_post_smp_prepare_boot_cpu();
@@ -577,10 +577,10 @@ clear_cpu_sibling_map(int cpu)
 {
        int i;
-        for_each_cpu_mask(i, per_cpu(cpu_sibling_map, cpu))
+        for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu))
-                cpu_clear(cpu, per_cpu(cpu_sibling_map, i));
+                cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i));
-        for_each_cpu_mask(i, cpu_core_map[cpu])
+        for_each_cpu(i, &cpu_core_map[cpu])
-                cpu_clear(cpu, cpu_core_map[i]);
+                cpumask_clear_cpu(cpu, &cpu_core_map[i]);
        per_cpu(cpu_sibling_map, cpu) = cpu_core_map[cpu] = CPU_MASK_NONE;
 }
@@ -592,12 +592,12 @@ remove_siblinginfo(int cpu)
        if (cpu_data(cpu)->threads_per_core == 1 &&
            cpu_data(cpu)->cores_per_socket == 1) {
-                cpu_clear(cpu, cpu_core_map[cpu]);
+                cpumask_clear_cpu(cpu, &cpu_core_map[cpu]);
-                cpu_clear(cpu, per_cpu(cpu_sibling_map, cpu));
+                cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, cpu));
                return;
        }
-        last = (cpus_weight(cpu_core_map[cpu]) == 1 ? 1 : 0);
+        last = (cpumask_weight(&cpu_core_map[cpu]) == 1 ? 1 : 0);
        /* remove it from all sibling map's */
        clear_cpu_sibling_map(cpu);
@@ -673,7 +673,7 @@ int __cpu_disable(void)
        remove_siblinginfo(cpu);
        fixup_irqs();
        local_flush_tlb_all();
-        cpu_clear(cpu, cpu_callin_map);
+        cpumask_clear_cpu(cpu, &cpu_callin_map);
        return 0;
 }
@@ -718,11 +718,13 @@ static inline void set_cpu_sibling_map(int cpu)
        for_each_online_cpu(i) {
                if ((cpu_data(cpu)->socket_id == cpu_data(i)->socket_id)) {
-                        cpu_set(i, cpu_core_map[cpu]);
+                        cpumask_set_cpu(i, &cpu_core_map[cpu]);
-                        cpu_set(cpu, cpu_core_map[i]);
+                        cpumask_set_cpu(cpu, &cpu_core_map[i]);
                        if (cpu_data(cpu)->core_id == cpu_data(i)->core_id) {
-                                cpu_set(i, per_cpu(cpu_sibling_map, cpu));
+                                cpumask_set_cpu(i,
-                                cpu_set(cpu, per_cpu(cpu_sibling_map, i));
+                                                &per_cpu(cpu_sibling_map, cpu));
+                                cpumask_set_cpu(cpu,
+                                                &per_cpu(cpu_sibling_map, i));
                        }
                }
        }
@@ -742,7 +744,7 @@ __cpu_up(unsigned int cpu, struct task_struct *tidle)
         * Already booted cpu? not valid anymore since we dont
         * do idle loop tightspin anymore.
         */
-        if (cpu_isset(cpu, cpu_callin_map))
+        if (cpumask_test_cpu(cpu, &cpu_callin_map))
                return -EINVAL;
        per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
@@ -753,8 +755,8 @@ __cpu_up(unsigned int cpu, struct task_struct *tidle)
        if (cpu_data(cpu)->threads_per_core == 1 &&
            cpu_data(cpu)->cores_per_socket == 1) {
-                cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
+                cpumask_set_cpu(cpu, &per_cpu(cpu_sibling_map, cpu));
-                cpu_set(cpu, cpu_core_map[cpu]);
+                cpumask_set_cpu(cpu, &cpu_core_map[cpu]);
                return 0;
        }
diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c
index 965ab42fabb0..c01fe8991244 100644
--- a/arch/ia64/kernel/topology.c
+++ b/arch/ia64/kernel/topology.c
@@ -148,7 +148,7 @@ static void cache_shared_cpu_map_setup(unsigned int cpu,
        if (cpu_data(cpu)->threads_per_core <= 1 &&
                cpu_data(cpu)->cores_per_socket <= 1) {
-                cpu_set(cpu, this_leaf->shared_cpu_map);
+                cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);
                return;
        }
@@ -164,7 +164,7 @@ static void cache_shared_cpu_map_setup(unsigned int cpu,
                        if (cpu_data(cpu)->socket_id == cpu_data(j)->socket_id
                                && cpu_data(j)->core_id == csi.log1_cid
                                && cpu_data(j)->thread_id == csi.log1_tid)
-                                cpu_set(j, this_leaf->shared_cpu_map);
+                                cpumask_set_cpu(j, &this_leaf->shared_cpu_map);
                i++;
        } while (i < num_shared &&
@@ -177,7 +177,7 @@ static void cache_shared_cpu_map_setup(unsigned int cpu,
 static void cache_shared_cpu_map_setup(unsigned int cpu,
                struct cache_info * this_leaf)
 {
-        cpu_set(cpu, this_leaf->shared_cpu_map);
+        cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);
        return;
 }
 #endif
diff --git a/arch/m32r/kernel/smpboot.c b/arch/m32r/kernel/smpboot.c
index bb21f4f63170..a468467542f4 100644
--- a/arch/m32r/kernel/smpboot.c
+++ b/arch/m32r/kernel/smpboot.c
@@ -376,7 +376,7 @@ void __init smp_cpus_done(unsigned int max_cpus)
        if (!cpumask_equal(&cpu_callin_map, cpu_online_mask))
                BUG();
-        for (cpu_id = 0 ; cpu_id < num_online_cpus() ; cpu_id++)
+        for_each_online_cpu(cpu_id)
                show_cpu_info(cpu_id);
        /*
diff --git a/arch/m68k/coldfire/m527x.c b/arch/m68k/coldfire/m527x.c
index 2ba470735bed..c0b3e28f91df 100644
--- a/arch/m68k/coldfire/m527x.c
+++ b/arch/m68k/coldfire/m527x.c
@@ -92,7 +92,6 @@ static void __init m527x_uarts_init(void)
 static void __init m527x_fec_init(void)
 {
-        u16 par;
        u8 v;
        /* Set multi-function pins to ethernet mode for fec0 */
@@ -100,6 +99,8 @@ static void __init m527x_fec_init(void)
        v = readb(MCFGPIO_PAR_FECI2C);
        writeb(v | 0xf0, MCFGPIO_PAR_FECI2C);
 #else
+        u16 par;
        par = readw(MCFGPIO_PAR_FECI2C);
        writew(par | 0xf00, MCFGPIO_PAR_FECI2C);
        v = readb(MCFGPIO_PAR_FEC0HL);
diff --git a/arch/m68k/include/asm/m527xsim.h b/arch/m68k/include/asm/m527xsim.h
index 1bebbe78055a..2c648a043f24 100644
--- a/arch/m68k/include/asm/m527xsim.h
+++ b/arch/m68k/include/asm/m527xsim.h
@@ -103,8 +103,10 @@
 */
 #define MCFFEC_BASE0            (MCF_IPSBAR + 0x1000)
 #define MCFFEC_SIZE0            0x800
+#ifdef CONFIG_M5275
 #define MCFFEC_BASE1            (MCF_IPSBAR + 0x1800)
 #define MCFFEC_SIZE1            0x800
+#endif
 /*
 *      QSPI module.
diff --git a/arch/m68k/include/asm/m68360_pram.h b/arch/m68k/include/asm/m68360_pram.h
index e6088bbce93d..c0cbd96f09bc 100644
--- a/arch/m68k/include/asm/m68360_pram.h
+++ b/arch/m68k/include/asm/m68360_pram.h
@@ -170,7 +170,7 @@ struct uart_pram {
    unsigned short  frmer;          /* Rx framing error counter */
    unsigned short  nosec;          /* Rx noise counter */
    unsigned short  brkec;          /* Rx break character counter */
-    unsigned short  brkln;          /* Reaceive break length */
+    unsigned short  brkln;          /* Receive break length */
                   
    unsigned short  uaddr1;         /* address character 1 */
    unsigned short  uaddr2;         /* address character 2 */
@@ -338,7 +338,7 @@ struct ethernet_pram {
    unsigned long   c_pres;         /* preset CRC */
    unsigned long   c_mask;         /* constant mask for CRC */
    unsigned long   crcec;          /* CRC error counter */
-    unsigned long   alec;           /* alighnment error counter */
+    unsigned long   alec;           /* alignment error counter */
    unsigned long   disfc;          /* discard frame counter */
    unsigned short  pads;           /* short frame PAD characters */
    unsigned short  ret_lim;        /* retry limit threshold */
diff --git a/arch/mips/bcm63xx/irq.c b/arch/mips/bcm63xx/irq.c
index b94bf44d8d8e..e3e808a6c542 100644
--- a/arch/mips/bcm63xx/irq.c
+++ b/arch/mips/bcm63xx/irq.c
@@ -58,9 +58,9 @@ static inline int enable_irq_for_cpu(int cpu, struct irq_data *d,
 #ifdef CONFIG_SMP
        if (m)
-                enable &= cpu_isset(cpu, *m);
+                enable &= cpumask_test_cpu(cpu, m);
        else if (irqd_affinity_was_set(d))
-                enable &= cpu_isset(cpu, *d->affinity);
+                enable &= cpumask_test_cpu(cpu, d->affinity);
 #endif
        return enable;
 }
diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c
index 8b1eeffa12ed..56f5d080ef9d 100644
--- a/arch/mips/cavium-octeon/smp.c
+++ b/arch/mips/cavium-octeon/smp.c
@@ -72,7 +72,7 @@ static inline void octeon_send_ipi_mask(const struct cpumask *mask,
 {
        unsigned int i;
-        for_each_cpu_mask(i, *mask)
+        for_each_cpu(i, mask)
                octeon_send_ipi_single(i, action);
 }
@@ -239,7 +239,7 @@ static int octeon_cpu_disable(void)
                return -ENOTSUPP;
        set_cpu_online(cpu, false);
-        cpu_clear(cpu, cpu_callin_map);
+        cpumask_clear_cpu(cpu, &cpu_callin_map);
        octeon_fixup_irqs();
        flush_cache_all();
diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h
index eacf865d21c2..bb02fac9b4fa 100644
--- a/arch/mips/include/asm/smp.h
+++ b/arch/mips/include/asm/smp.h
@@ -88,7 +88,7 @@ static inline void arch_send_call_function_single_ipi(int cpu)
 {
        extern struct plat_smp_ops *mp_ops;     /* private */
-        mp_ops->send_ipi_mask(&cpumask_of_cpu(cpu), SMP_CALL_FUNCTION);
+        mp_ops->send_ipi_mask(cpumask_of(cpu), SMP_CALL_FUNCTION);
 }
 static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
diff --git a/arch/mips/kernel/crash.c b/arch/mips/kernel/crash.c
index d21264681e97..d434d5d5ae6e 100644
--- a/arch/mips/kernel/crash.c
+++ b/arch/mips/kernel/crash.c
@@ -25,9 +25,9 @@ static void crash_shutdown_secondary(void *ignore)
                return;
        local_irq_disable();
-        if (!cpu_isset(cpu, cpus_in_crash))
+        if (!cpumask_test_cpu(cpu, &cpus_in_crash))
                crash_save_cpu(regs, cpu);
-        cpu_set(cpu, cpus_in_crash);
+        cpumask_set_cpu(cpu, &cpus_in_crash);
        while (!atomic_read(&kexec_ready_to_reboot))
                cpu_relax();
@@ -50,7 +50,7 @@ static void crash_kexec_prepare_cpus(void)
         */
        pr_emerg("Sending IPI to other cpus...\n");
        msecs = 10000;
-        while ((cpus_weight(cpus_in_crash) < ncpus) && (--msecs > 0)) {
+        while ((cpumask_weight(&cpus_in_crash) < ncpus) && (--msecs > 0)) {
                cpu_relax();
                mdelay(1);
        }
@@ -66,5 +66,5 @@ void default_machine_crash_shutdown(struct pt_regs *regs)
        crashing_cpu = smp_processor_id();
        crash_save_cpu(regs, crashing_cpu);
        crash_kexec_prepare_cpus();
-        cpu_set(crashing_cpu, cpus_in_crash);
+        cpumask_set_cpu(crashing_cpu, &cpus_in_crash);
 }
diff --git a/arch/mips/kernel/mips-mt-fpaff.c b/arch/mips/kernel/mips-mt-fpaff.c
index 362bb3707e62..3e4491aa6d6b 100644
--- a/arch/mips/kernel/mips-mt-fpaff.c
+++ b/arch/mips/kernel/mips-mt-fpaff.c
@@ -114,8 +114,8 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len,
        /* Compute new global allowed CPU set if necessary */
        ti = task_thread_info(p);
        if (test_ti_thread_flag(ti, TIF_FPUBOUND) &&
-            cpus_intersects(*new_mask, mt_fpu_cpumask)) {
+            cpumask_intersects(new_mask, &mt_fpu_cpumask)) {
-                cpus_and(*effective_mask, *new_mask, mt_fpu_cpumask);
+                cpumask_and(effective_mask, new_mask, &mt_fpu_cpumask);
                retval = set_cpus_allowed_ptr(p, effective_mask);
        } else {
                cpumask_copy(effective_mask, new_mask);
diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c
index d295bd1e4996..f2975d4d1e44 100644
--- a/arch/mips/kernel/process.c
+++ b/arch/mips/kernel/process.c
@@ -49,7 +49,7 @@
 void arch_cpu_idle_dead(void)
 {
        /* What the heck is this check doing ? */
-        if (!cpu_isset(smp_processor_id(), cpu_callin_map))
+        if (!cpumask_test_cpu(smp_processor_id(), &cpu_callin_map))
                play_dead();
 }
 #endif
diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c
index b8bd9340c9c7..fd528d7ea278 100644
--- a/arch/mips/kernel/smp-bmips.c
+++ b/arch/mips/kernel/smp-bmips.c
@@ -362,7 +362,7 @@ static int bmips_cpu_disable(void)
        pr_info("SMP: CPU%d is offline\n", cpu);
        set_cpu_online(cpu, false);
-        cpu_clear(cpu, cpu_callin_map);
+        cpumask_clear_cpu(cpu, &cpu_callin_map);
        clear_c0_status(IE_IRQ5);
        local_flush_tlb_all();
diff --git a/arch/mips/kernel/smp-cmp.c b/arch/mips/kernel/smp-cmp.c
index e36a859af666..d5e0f949dc48 100644
--- a/arch/mips/kernel/smp-cmp.c
+++ b/arch/mips/kernel/smp-cmp.c
@@ -66,7 +66,7 @@ static void cmp_smp_finish(void)
 #ifdef CONFIG_MIPS_MT_FPAFF
        /* If we have an FPU, enroll ourselves in the FPU-full mask */
        if (cpu_has_fpu)
-                cpu_set(smp_processor_id(), mt_fpu_cpumask);
+                cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask);
 #endif /* CONFIG_MIPS_MT_FPAFF */
        local_irq_enable();
@@ -110,7 +110,7 @@ void __init cmp_smp_setup(void)
 #ifdef CONFIG_MIPS_MT_FPAFF
        /* If we have an FPU, enroll ourselves in the FPU-full mask */
        if (cpu_has_fpu)
-                cpu_set(0, mt_fpu_cpumask);
+                cpumask_set_cpu(0, &mt_fpu_cpumask);
 #endif /* CONFIG_MIPS_MT_FPAFF */
        for (i = 1; i < NR_CPUS; i++) {
diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c
index d5589bedd0a4..7e011f95bb8e 100644
--- a/arch/mips/kernel/smp-cps.c
+++ b/arch/mips/kernel/smp-cps.c
@@ -290,7 +290,7 @@ static void cps_smp_finish(void)
 #ifdef CONFIG_MIPS_MT_FPAFF
        /* If we have an FPU, enroll ourselves in the FPU-full mask */
        if (cpu_has_fpu)
-                cpu_set(smp_processor_id(), mt_fpu_cpumask);
+                cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask);
 #endif /* CONFIG_MIPS_MT_FPAFF */
        local_irq_enable();
@@ -313,7 +313,7 @@ static int cps_cpu_disable(void)
        atomic_sub(1 << cpu_vpe_id(&current_cpu_data), &core_cfg->vpe_mask);
        smp_mb__after_atomic();
        set_cpu_online(cpu, false);
-        cpu_clear(cpu, cpu_callin_map);
+        cpumask_clear_cpu(cpu, &cpu_callin_map);
        return 0;
 }
diff --git a/arch/mips/kernel/smp-mt.c b/arch/mips/kernel/smp-mt.c
index 17ea705f6c40..86311a164ef1 100644
--- a/arch/mips/kernel/smp-mt.c
+++ b/arch/mips/kernel/smp-mt.c
@@ -178,7 +178,7 @@ static void vsmp_smp_finish(void)
 #ifdef CONFIG_MIPS_MT_FPAFF
        /* If we have an FPU, enroll ourselves in the FPU-full mask */
        if (cpu_has_fpu)
-                cpu_set(smp_processor_id(), mt_fpu_cpumask);
+                cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask);
 #endif /* CONFIG_MIPS_MT_FPAFF */
        local_irq_enable();
@@ -239,7 +239,7 @@ static void __init vsmp_smp_setup(void)
 #ifdef CONFIG_MIPS_MT_FPAFF
        /* If we have an FPU, enroll ourselves in the FPU-full mask */
        if (cpu_has_fpu)
-                cpu_set(0, mt_fpu_cpumask);
+                cpumask_set_cpu(0, &mt_fpu_cpumask);
 #endif /* CONFIG_MIPS_MT_FPAFF */
        if (!cpu_has_mipsmt)
                return;
diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
index 5b020bda3e05..193ace7955fb 100644
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -75,30 +75,30 @@ static inline void set_cpu_sibling_map(int cpu)
 {
        int i;
-        cpu_set(cpu, cpu_sibling_setup_map);
+        cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
        if (smp_num_siblings > 1) {
-                for_each_cpu_mask(i, cpu_sibling_setup_map) {
+                for_each_cpu(i, &cpu_sibling_setup_map) {
                        if (cpu_data[cpu].package == cpu_data[i].package &&
                                    cpu_data[cpu].core == cpu_data[i].core) {
-                                cpu_set(i, cpu_sibling_map[cpu]);
+                                cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
-                                cpu_set(cpu, cpu_sibling_map[i]);
+                                cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
                        }
                }
        } else
-                cpu_set(cpu, cpu_sibling_map[cpu]);
+                cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]);
 }
 static inline void set_cpu_core_map(int cpu)
 {
        int i;
-        cpu_set(cpu, cpu_core_setup_map);
+        cpumask_set_cpu(cpu, &cpu_core_setup_map);
-        for_each_cpu_mask(i, cpu_core_setup_map) {
+        for_each_cpu(i, &cpu_core_setup_map) {
                if (cpu_data[cpu].package == cpu_data[i].package) {
-                        cpu_set(i, cpu_core_map[cpu]);
+                        cpumask_set_cpu(i, &cpu_core_map[cpu]);
-                        cpu_set(cpu, cpu_core_map[i]);
+                        cpumask_set_cpu(cpu, &cpu_core_map[i]);
                }
        }
 }
@@ -138,7 +138,7 @@ asmlinkage void start_secondary(void)
        cpu = smp_processor_id();
        cpu_data[cpu].udelay_val = loops_per_jiffy;
-        cpu_set(cpu, cpu_coherent_mask);
+        cpumask_set_cpu(cpu, &cpu_coherent_mask);
        notify_cpu_starting(cpu);
        set_cpu_online(cpu, true);
@@ -146,7 +146,7 @@ asmlinkage void start_secondary(void)
        set_cpu_sibling_map(cpu);
        set_cpu_core_map(cpu);
-        cpu_set(cpu, cpu_callin_map);
+        cpumask_set_cpu(cpu, &cpu_callin_map);
        synchronise_count_slave(cpu);
@@ -208,7 +208,7 @@ void smp_prepare_boot_cpu(void)
 {
        set_cpu_possible(0, true);
        set_cpu_online(0, true);
-        cpu_set(0, cpu_callin_map);
+        cpumask_set_cpu(0, &cpu_callin_map);
 }
 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
@@ -218,7 +218,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
        /*
         * Trust is futile.  We should really have timeouts ...
         */
-        while (!cpu_isset(cpu, cpu_callin_map))
+        while (!cpumask_test_cpu(cpu, &cpu_callin_map))
                udelay(100);
        synchronise_count_master(cpu);
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index e334c641a81b..ba32e48d4697 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -1153,13 +1153,13 @@ static void mt_ase_fp_affinity(void)
                 * restricted the allowed set to exclude any CPUs with FPUs,
                 * we'll skip the procedure.
                 */
-                if (cpus_intersects(current->cpus_allowed, mt_fpu_cpumask)) {
+                if (cpumask_intersects(&current->cpus_allowed, &mt_fpu_cpumask)) {
                        cpumask_t tmask;
                        current->thread.user_cpus_allowed
                                = current->cpus_allowed;
-                        cpus_and(tmask, current->cpus_allowed,
+                        cpumask_and(&tmask, &current->cpus_allowed,
-                                mt_fpu_cpumask);
+                                    &mt_fpu_cpumask);
                        set_cpus_allowed_ptr(current, &tmask);
                        set_thread_flag(TIF_FPUBOUND);
                }
diff --git a/arch/mips/loongson/loongson-3/numa.c b/arch/mips/loongson/loongson-3/numa.c
index 6cae0e75de27..12d14ed48778 100644
--- a/arch/mips/loongson/loongson-3/numa.c
+++ b/arch/mips/loongson/loongson-3/numa.c
@@ -233,7 +233,7 @@ static __init void prom_meminit(void)
                if (node_online(node)) {
                        szmem(node);
                        node_mem_init(node);
-                        cpus_clear(__node_data[(node)]->cpumask);
+                        cpumask_clear(&__node_data[(node)]->cpumask);
                }
        }
        for (cpu = 0; cpu < loongson_sysconf.nr_cpus; cpu++) {
@@ -244,7 +244,7 @@ static __init void prom_meminit(void)
                if (loongson_sysconf.reserved_cpus_mask & (1<<cpu))
                        continue;
-                cpu_set(active_cpu, __node_data[(node)]->cpumask);
+                cpumask_set_cpu(active_cpu, &__node_data[(node)]->cpumask);
                pr_info("NUMA: set cpumask cpu %d on node %d\n", active_cpu, node);
                active_cpu++;
diff --git a/arch/mips/loongson/loongson-3/smp.c b/arch/mips/loongson/loongson-3/smp.c
index e2eb688b5434..e3c68b5da18d 100644
--- a/arch/mips/loongson/loongson-3/smp.c
+++ b/arch/mips/loongson/loongson-3/smp.c
@@ -408,7 +408,7 @@ static int loongson3_cpu_disable(void)
                return -EBUSY;
        set_cpu_online(cpu, false);
-        cpu_clear(cpu, cpu_callin_map);
+        cpumask_clear_cpu(cpu, &cpu_callin_map);
        local_irq_save(flags);
        fixup_irqs();
        local_irq_restore(flags);
diff --git a/arch/mips/paravirt/paravirt-smp.c b/arch/mips/paravirt/paravirt-smp.c
index 0164b0c48352..42181c7105df 100644
--- a/arch/mips/paravirt/paravirt-smp.c
+++ b/arch/mips/paravirt/paravirt-smp.c
@@ -75,7 +75,7 @@ static void paravirt_send_ipi_mask(const struct cpumask *mask, unsigned int acti
 {
        unsigned int cpu;
-        for_each_cpu_mask(cpu, *mask)
+        for_each_cpu(cpu, mask)
                paravirt_send_ipi_single(cpu, action);
 }
diff --git a/arch/mips/sgi-ip27/ip27-init.c b/arch/mips/sgi-ip27/ip27-init.c
index ee736bd103f8..570098bfdf87 100644
--- a/arch/mips/sgi-ip27/ip27-init.c
+++ b/arch/mips/sgi-ip27/ip27-init.c
@@ -60,7 +60,7 @@ static void per_hub_init(cnodeid_t cnode)
        nasid_t nasid = COMPACT_TO_NASID_NODEID(cnode);
        int i;
-        cpu_set(smp_processor_id(), hub->h_cpus);
+        cpumask_set_cpu(smp_processor_id(), &hub->h_cpus);
        if (test_and_set_bit(cnode, hub_init_mask))
                return;
diff --git a/arch/mips/sgi-ip27/ip27-klnuma.c b/arch/mips/sgi-ip27/ip27-klnuma.c
index ecbb62f339c5..bda90cf87e8c 100644
--- a/arch/mips/sgi-ip27/ip27-klnuma.c
+++ b/arch/mips/sgi-ip27/ip27-klnuma.c
@@ -29,8 +29,8 @@ static cpumask_t ktext_repmask;
 void __init setup_replication_mask(void)
 {
        /* Set only the master cnode's bit.  The master cnode is always 0. */
-        cpus_clear(ktext_repmask);
+        cpumask_clear(&ktext_repmask);
-        cpu_set(0, ktext_repmask);
+        cpumask_set_cpu(0, &ktext_repmask);
 #ifdef CONFIG_REPLICATE_KTEXT
 #ifndef CONFIG_MAPPED_KERNEL
@@ -43,7 +43,7 @@ void __init setup_replication_mask(void)
                        if (cnode == 0)
                                continue;
                        /* Advertise that we have a copy of the kernel */
-                        cpu_set(cnode, ktext_repmask);
+                        cpumask_set_cpu(cnode, &ktext_repmask);
                }
        }
 #endif
@@ -99,7 +99,7 @@ void __init replicate_kernel_text()
                client_nasid = COMPACT_TO_NASID_NODEID(cnode);
                /* Check if this node should get a copy of the kernel */
-                if (cpu_isset(cnode, ktext_repmask)) {
+                if (cpumask_test_cpu(cnode, &ktext_repmask)) {
                        server_nasid = client_nasid;
                        copy_kernel(server_nasid);
                }
@@ -124,7 +124,7 @@ unsigned long node_getfirstfree(cnodeid_t cnode)
        loadbase += 16777216;
 #endif
        offset = PAGE_ALIGN((unsigned long)(&_end)) - loadbase;
-        if ((cnode == 0) || (cpu_isset(cnode, ktext_repmask)))
+        if ((cnode == 0) || (cpumask_test_cpu(cnode, &ktext_repmask)))
                return TO_NODE(nasid, offset) >> PAGE_SHIFT;
        else
                return KDM_TO_PHYS(PAGE_ALIGN(SYMMON_STK_ADDR(nasid, 0))) >> PAGE_SHIFT;
diff --git a/arch/mips/sgi-ip27/ip27-memory.c b/arch/mips/sgi-ip27/ip27-memory.c
index 0b68469e063f..8d0eb2643248 100644
--- a/arch/mips/sgi-ip27/ip27-memory.c
+++ b/arch/mips/sgi-ip27/ip27-memory.c
@@ -404,7 +404,7 @@ static void __init node_mem_init(cnodeid_t node)
        NODE_DATA(node)->node_start_pfn = start_pfn;
        NODE_DATA(node)->node_spanned_pages = end_pfn - start_pfn;
-        cpus_clear(hub_data(node)->h_cpus);
+        cpumask_clear(&hub_data(node)->h_cpus);
        slot_freepfn += PFN_UP(sizeof(struct pglist_data) +
                               sizeof(struct hub_data));
diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c
index cfe056fe7f5c..f3191db6e2e9 100644
--- a/arch/parisc/kernel/irq.c
+++ b/arch/parisc/kernel/irq.c
@@ -525,8 +525,8 @@ void do_cpu_irq_mask(struct pt_regs *regs)
        desc = irq_to_desc(irq);
        cpumask_copy(&dest, desc->irq_data.affinity);
        if (irqd_is_per_cpu(&desc->irq_data) &&
-            !cpu_isset(smp_processor_id(), dest)) {
+            !cpumask_test_cpu(smp_processor_id(), &dest)) {
-                int cpu = first_cpu(dest);
+                int cpu = cpumask_first(&dest);
                printk(KERN_DEBUG "redirecting irq %d from CPU %d to %d\n",
                       irq, smp_processor_id(), cpu);
diff --git a/arch/powerpc/include/asm/cputhreads.h b/arch/powerpc/include/asm/cputhreads.h
index 4c8ad592ae33..5be6c4753667 100644
--- a/arch/powerpc/include/asm/cputhreads.h
+++ b/arch/powerpc/include/asm/cputhreads.h
@@ -25,7 +25,7 @@ extern cpumask_t threads_core_mask;
 #define threads_per_core        1
 #define threads_per_subcore     1
 #define threads_shift           0
-#define threads_core_mask       (CPU_MASK_CPU0)
+#define threads_core_mask       (*get_cpu_mask(0))
 #endif
 /* cpu_thread_mask_to_cores - Return a cpumask of one per cores
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index de2726a487b0..8e58c614c37d 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -115,7 +115,7 @@ config S390
        select HAVE_ARCH_SECCOMP_FILTER
        select HAVE_ARCH_TRACEHOOK
        select HAVE_ARCH_TRANSPARENT_HUGEPAGE
-        select HAVE_BPF_JIT if PACK_STACK
+        select HAVE_BPF_JIT if PACK_STACK && HAVE_MARCH_Z9_109_FEATURES
        select HAVE_CMPXCHG_DOUBLE
        select HAVE_CMPXCHG_LOCAL
        select HAVE_DEBUG_KMEMLEAK
diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h
index 709955ddaa4d..9d395961e713 100644
--- a/arch/s390/include/asm/dma-mapping.h
+++ b/arch/s390/include/asm/dma-mapping.h
@@ -42,7 +42,7 @@ static inline int dma_supported(struct device *dev, u64 mask)
 static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
 {
        if (!dev->dma_mask)
-                return 0;
+                return false;
        return addr + size - 1 <= *dev->dma_mask;
 }
diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index ef803c202d42..a648338c434a 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -7,6 +7,7 @@
 #define PCI_BAR_COUNT   6
 #include <linux/pci.h>
+#include <linux/mutex.h>
 #include <asm-generic/pci.h>
 #include <asm-generic/pci-dma-compat.h>
 #include <asm/pci_clp.h>
@@ -44,10 +45,6 @@ struct zpci_fmb {
        u64 rpcit_ops;
        u64 dma_rbytes;
        u64 dma_wbytes;
-        /* software counters */
-        atomic64_t allocated_pages;
-        atomic64_t mapped_pages;
-        atomic64_t unmapped_pages;
 } __packed __aligned(16);
 enum zpci_state {
@@ -80,6 +77,7 @@ struct zpci_dev {
        u8              pft;            /* pci function type */
        u16             domain;
+        struct mutex lock;
        u8 pfip[CLP_PFIP_NR_SEGMENTS];  /* pci function internal path */
        u32 uid;                        /* user defined id */
        u8 util_str[CLP_UTIL_STR_LEN];  /* utility string */
@@ -111,6 +109,10 @@ struct zpci_dev {
        /* Function measurement block */
        struct zpci_fmb *fmb;
        u16             fmb_update;     /* update interval */
+        /* software counters */
+        atomic64_t allocated_pages;
+        atomic64_t mapped_pages;
+        atomic64_t unmapped_pages;
        enum pci_bus_speed max_bus_speed;
diff --git a/arch/s390/net/bpf_jit.S b/arch/s390/net/bpf_jit.S
index ba44c9f55346..a1c917d881ec 100644
--- a/arch/s390/net/bpf_jit.S
+++ b/arch/s390/net/bpf_jit.S
@@ -1,134 +1,115 @@
 /*
 * BPF Jit compiler for s390, help functions.
 *
- * Copyright IBM Corp. 2012
+ * Copyright IBM Corp. 2012,2015
 *
 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *            Michael Holzheu <holzheu@linux.vnet.ibm.com>
 */
 #include <linux/linkage.h>
+#include "bpf_jit.h"
 /*
 * Calling convention:
- * registers %r2, %r6-%r8, %r10-%r11, %r13, %r15 are call saved
+ * registers %r7-%r10, %r11,%r13, and %r15 are call saved
- *   %r2: skb pointer
+ *
- *   %r3: offset parameter
+ * Input (64 bit):
- *   %r5: BPF A accumulator
+ *   %r3 (%b2) = offset into skb data
- *   %r8: return address
+ *   %r6 (%b5) = return address
- *   %r9: save register for skb pointer
+ *   %r7 (%b6) = skb pointer
- *   %r10: skb->data
+ *   %r12      = skb data pointer
- *   %r11: skb->len - skb->data_len (headlen)
+ *
- *   %r12: BPF X accumulator
+ * Output:
+ *   %r14= %b0 = return value (read skb value)
+ *
+ * Work registers: %r2,%r4,%r5,%r14
 *
 * skb_copy_bits takes 4 parameters:
 *   %r2 = skb pointer
 *   %r3 = offset into skb data
 *   %r4 = pointer to temp buffer
 *   %r5 = length to copy
+ *   Return value in %r2: 0 = ok
+ *
+ * bpf_internal_load_pointer_neg_helper takes 3 parameters:
+ *   %r2 = skb pointer
+ *   %r3 = offset into data
+ *   %r4 = length to copy
+ *   Return value in %r2: Pointer to data
 */
-#define SKBDATA %r8
-        /* A = *(u32 *) (skb->data+K+X) */
-ENTRY(sk_load_word_ind)
-        ar      %r3,%r12                # offset += X
-        bmr     %r8                     # < 0 -> return with cc
-        /* A = *(u32 *) (skb->data+K) */
-ENTRY(sk_load_word)
-        llgfr   %r1,%r3                 # extend offset
-        ahi     %r3,4                   # offset + 4
-        clr     %r11,%r3                # hlen <= offset + 4 ?
-        jl      sk_load_word_slow
-        l       %r5,0(%r1,%r10)         # get word from skb
-        xr      %r1,%r1                 # set cc to zero
-        br      %r8
-sk_load_word_slow:
+#define SKF_MAX_NEG_OFF -0x200000       /* SKF_LL_OFF from filter.h */
-        lgr     %r9,%r2                 # save %r2
-        lgr     %r3,%r1                 # offset
-        la      %r4,160(%r15)           # pointer to temp buffer
-        lghi    %r5,4                   # 4 bytes
-        brasl   %r14,skb_copy_bits      # get data from skb
-        l       %r5,160(%r15)           # load result from temp buffer
-        ltgr    %r2,%r2                 # set cc to (%r2 != 0)
-        lgr     %r2,%r9                 # restore %r2
-        br      %r8
-        /* A = *(u16 *) (skb->data+K+X) */
+/*
-ENTRY(sk_load_half_ind)
+ * Load SIZE bytes from SKB
-        ar      %r3,%r12                # offset += X
+ */
-        bmr     %r8                     # < 0 -> return with cc
+#define sk_load_common(NAME, SIZE, LOAD)                                \
+ENTRY(sk_load_##NAME);                                                  \
-        /* A = *(u16 *) (skb->data+K) */
+        ltgr    %r3,%r3;                /* Is offset negative? */       \
-ENTRY(sk_load_half)
+        jl      sk_load_##NAME##_slow_neg;                              \
-        llgfr   %r1,%r3                 # extend offset
+ENTRY(sk_load_##NAME##_pos);                                            \
-        ahi     %r3,2                   # offset + 2
+        aghi    %r3,SIZE;               /* Offset + SIZE */             \
-        clr     %r11,%r3                # hlen <= offset + 2 ?
+        clg     %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */     \
-        jl      sk_load_half_slow
+        jh      sk_load_##NAME##_slow;                                  \
-        llgh    %r5,0(%r1,%r10)         # get half from skb
+        LOAD    %r14,-SIZE(%r3,%r12);   /* Get data from skb */         \
-        xr      %r1,%r1                 # set cc to zero
+        b       OFF_OK(%r6);            /* Return */                    \
-        br      %r8
+                                                                        \
+sk_load_##NAME##_slow:;                                                 \
-sk_load_half_slow:
+        lgr     %r2,%r7;                /* Arg1 = skb pointer */        \
-        lgr     %r9,%r2                 # save %r2
+        aghi    %r3,-SIZE;              /* Arg2 = offset */             \
-        lgr     %r3,%r1                 # offset
+        la      %r4,STK_OFF_TMP(%r15);  /* Arg3 = temp bufffer */       \
-        la      %r4,162(%r15)           # pointer to temp buffer
+        lghi    %r5,SIZE;               /* Arg4 = size */               \
-        lghi    %r5,2                   # 2 bytes
+        brasl   %r14,skb_copy_bits;     /* Get data from skb */         \
-        brasl   %r14,skb_copy_bits      # get data from skb
+        LOAD    %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */    \
-        xc      160(2,%r15),160(%r15)
+        ltgr    %r2,%r2;                /* Set cc to (%r2 != 0) */      \
-        l       %r5,160(%r15)           # load result from temp buffer
+        br      %r6;                    /* Return */
-        ltgr    %r2,%r2                 # set cc to (%r2 != 0)
-        lgr     %r2,%r9                 # restore %r2
-        br      %r8
-        /* A = *(u8 *) (skb->data+K+X) */
+sk_load_common(word, 4, llgf)   /* r14 = *(u32 *) (skb->data+offset) */
-ENTRY(sk_load_byte_ind)
+sk_load_common(half, 2, llgh)   /* r14 = *(u16 *) (skb->data+offset) */
-        ar      %r3,%r12                # offset += X
-        bmr     %r8                     # < 0 -> return with cc
-        /* A = *(u8 *) (skb->data+K) */
+/*
+ * Load 1 byte from SKB (optimized version)
+ */
+        /* r14 = *(u8 *) (skb->data+offset) */
 ENTRY(sk_load_byte)
-        llgfr   %r1,%r3                 # extend offset
+        ltgr    %r3,%r3                 # Is offset negative?
-        clr     %r11,%r3                # hlen < offset ?
+        jl      sk_load_byte_slow_neg
-        jle     sk_load_byte_slow
+ENTRY(sk_load_byte_pos)
-        lhi     %r5,0
+        clg     %r3,STK_OFF_HLEN(%r15)  # Offset >= hlen?
-        ic      %r5,0(%r1,%r10)         # get byte from skb
+        jnl     sk_load_byte_slow
-        xr      %r1,%r1                 # set cc to zero
+        llgc    %r14,0(%r3,%r12)        # Get byte from skb
-        br      %r8
+        b       OFF_OK(%r6)             # Return OK
 sk_load_byte_slow:
-        lgr     %r9,%r2                 # save %r2
+        lgr     %r2,%r7                 # Arg1 = skb pointer
-        lgr     %r3,%r1                 # offset
+                                        # Arg2 = offset
-        la      %r4,163(%r15)           # pointer to temp buffer
+        la      %r4,STK_OFF_TMP(%r15)   # Arg3 = pointer to temp buffer
-        lghi    %r5,1                   # 1 byte
+        lghi    %r5,1                   # Arg4 = size (1 byte)
-        brasl   %r14,skb_copy_bits      # get data from skb
+        brasl   %r14,skb_copy_bits      # Get data from skb
-        xc      160(3,%r15),160(%r15)
+        llgc    %r14,STK_OFF_TMP(%r15)  # Load result from temp buffer
-        l       %r5,160(%r15)           # load result from temp buffer
+        ltgr    %r2,%r2                 # Set cc to (%r2 != 0)
-        ltgr    %r2,%r2                 # set cc to (%r2 != 0)
+        br      %r6                     # Return cc
-        lgr     %r2,%r9                 # restore %r2
-        br      %r8
+#define sk_negative_common(NAME, SIZE, LOAD)                            \
+sk_load_##NAME##_slow_neg:;                                             \
+        cgfi    %r3,SKF_MAX_NEG_OFF;                                    \
+        jl      bpf_error;                                              \
+        lgr     %r2,%r7;                /* Arg1 = skb pointer */        \
+                                        /* Arg2 = offset */             \
+        lghi    %r4,SIZE;               /* Arg3 = size */               \
+        brasl   %r14,bpf_internal_load_pointer_neg_helper;              \
+        ltgr    %r2,%r2;                                                \
+        jz      bpf_error;                                              \
+        LOAD    %r14,0(%r2);            /* Get data from pointer */     \
+        xr      %r3,%r3;                /* Set cc to zero */            \
+        br      %r6;                    /* Return cc */
-        /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */
+sk_negative_common(word, 4, llgf)
-ENTRY(sk_load_byte_msh)
+sk_negative_common(half, 2, llgh)
-        llgfr   %r1,%r3                 # extend offset
+sk_negative_common(byte, 1, llgc)
-        clr     %r11,%r3                # hlen < offset ?
-        jle     sk_load_byte_msh_slow
-        lhi     %r12,0
-        ic      %r12,0(%r1,%r10)        # get byte from skb
-        nill    %r12,0x0f
-        sll     %r12,2
-        xr      %r1,%r1                 # set cc to zero
-        br      %r8
-sk_load_byte_msh_slow:
+bpf_error:
-        lgr     %r9,%r2                 # save %r2
+# force a return 0 from jit handler
-        lgr     %r3,%r1                 # offset
+        ltgr    %r15,%r15       # Set condition code
-        la      %r4,163(%r15)           # pointer to temp buffer
+        br      %r6
-        lghi    %r5,1                   # 1 byte
-        brasl   %r14,skb_copy_bits      # get data from skb
-        xc      160(3,%r15),160(%r15)
-        l       %r12,160(%r15)          # load result from temp buffer
-        nill    %r12,0x0f
-        sll     %r12,2
-        ltgr    %r2,%r2                 # set cc to (%r2 != 0)
-        lgr     %r2,%r9                 # restore %r2
-        br      %r8
diff --git a/arch/s390/net/bpf_jit.h b/arch/s390/net/bpf_jit.h
new file mode 100644
index 000000000000..ba8593a515ba
--- /dev/null
+++ b/arch/s390/net/bpf_jit.h
@@ -0,0 +1,58 @@
+/*
+ * BPF Jit compiler defines
+ *
+ * Copyright IBM Corp. 2012,2015
+ *
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *            Michael Holzheu <holzheu@linux.vnet.ibm.com>
+ */
+#ifndef __ARCH_S390_NET_BPF_JIT_H
+#define __ARCH_S390_NET_BPF_JIT_H
+#ifndef __ASSEMBLY__
+#include <linux/filter.h>
+#include <linux/types.h>
+extern u8 sk_load_word_pos[], sk_load_half_pos[], sk_load_byte_pos[];
+extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
+#endif /* __ASSEMBLY__ */
+/*
+ * Stackframe layout (packed stack):
+ *
+ *                                  ^ high
+ *            +---------------+     |
+ *            | old backchain |     |
+ *            +---------------+     |
+ *            |   r15 - r6    |     |
+ * BFP     -> +===============+     |
+ *            |               |     |
+ *            |   BPF stack   |     |
+ *            |               |     |
+ *            +---------------+     |
+ *            | 8 byte hlen   |     |
+ * R15+168 -> +---------------+     |
+ *            | 4 byte align  |     |
+ *            +---------------+     |
+ *            | 4 byte temp   |     |
+ *            | for bpf_jit.S |     |
+ * R15+160 -> +---------------+     |
+ *            | new backchain |     |
+ * R15+152 -> +---------------+     |
+ *            | + 152 byte SA |     |
+ * R15     -> +---------------+     + low
+ *
+ * We get 160 bytes stack space from calling function, but only use
+ * 11 * 8 byte (old backchain + r15 - r6) for storing registers.
+ */
+#define STK_OFF (MAX_BPF_STACK + 8 + 4 + 4 + (160 - 11 * 8))
+#define STK_OFF_TMP     160     /* Offset of tmp buffer on stack */
+#define STK_OFF_HLEN    168     /* Offset of SKB header length on stack */
+/* Offset to skip condition code check */
+#define OFF_OK          4
+#endif /* __ARCH_S390_NET_BPF_JIT_H */
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index bbd1981cc150..7690dc8e1ab5 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -1,817 +1,1209 @@
 /*
 * BPF Jit compiler for s390.
 *
- * Copyright IBM Corp. 2012
+ * Minimum build requirements:
+ *
+ *  - HAVE_MARCH_Z196_FEATURES: laal, laalg
+ *  - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj
+ *  - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf
+ *  - PACK_STACK
+ *  - 64BIT
+ *
+ * Copyright IBM Corp. 2012,2015
 *
 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *            Michael Holzheu <holzheu@linux.vnet.ibm.com>
 */
+#define KMSG_COMPONENT "bpf_jit"
+#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 #include <linux/netdevice.h>
-#include <linux/if_vlan.h>
 #include <linux/filter.h>
 #include <linux/init.h>
 #include <asm/cacheflush.h>
-#include <asm/facility.h>
 #include <asm/dis.h>
+#include "bpf_jit.h"
-/*
- * Conventions:
- *   %r2 = skb pointer
- *   %r3 = offset parameter
- *   %r4 = scratch register / length parameter
- *   %r5 = BPF A accumulator
- *   %r8 = return address
- *   %r9 = save register for skb pointer
- *   %r10 = skb->data
- *   %r11 = skb->len - skb->data_len (headlen)
- *   %r12 = BPF X accumulator
- *   %r13 = literal pool pointer
- *   0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS
- */
 int bpf_jit_enable __read_mostly;
+struct bpf_jit {
+        u32 seen;               /* Flags to remember seen eBPF instructions */
+        u32 seen_reg[16];       /* Array to remember which registers are used */
+        u32 *addrs;             /* Array with relative instruction addresses */
+        u8 *prg_buf;            /* Start of program */
+        int size;               /* Size of program and literal pool */
+        int size_prg;           /* Size of program */
+        int prg;                /* Current position in program */
+        int lit_start;          /* Start of literal pool */
+        int lit;                /* Current position in literal pool */
+        int base_ip;            /* Base address for literal pool */
+        int ret0_ip;            /* Address of return 0 */
+        int exit_ip;            /* Address of exit */
+};
+#define BPF_SIZE_MAX    4096    /* Max size for program */
+#define SEEN_SKB        1       /* skb access */
+#define SEEN_MEM        2       /* use mem[] for temporary storage */
+#define SEEN_RET0       4       /* ret0_ip points to a valid return 0 */
+#define SEEN_LITERAL    8       /* code uses literals */
+#define SEEN_FUNC       16      /* calls C functions */
+#define SEEN_STACK      (SEEN_FUNC | SEEN_MEM | SEEN_SKB)
 /*
- * assembly code in arch/x86/net/bpf_jit.S
+ * s390 registers
 */
-extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[];
+#define REG_W0          (__MAX_BPF_REG+0)       /* Work register 1 (even) */
-extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[];
+#define REG_W1          (__MAX_BPF_REG+1)       /* Work register 2 (odd) */
+#define REG_SKB_DATA    (__MAX_BPF_REG+2)       /* SKB data register */
+#define REG_L           (__MAX_BPF_REG+3)       /* Literal pool register */
+#define REG_15          (__MAX_BPF_REG+4)       /* Register 15 */
+#define REG_0           REG_W0                  /* Register 0 */
+#define REG_2           BPF_REG_1               /* Register 2 */
+#define REG_14          BPF_REG_0               /* Register 14 */
-struct bpf_jit {
+/*
-        unsigned int seen;
+ * Mapping of BPF registers to s390 registers
-        u8 *start;
+ */
-        u8 *prg;
+static const int reg2hex[] = {
-        u8 *mid;
+        /* Return code */
-        u8 *lit;
+        [BPF_REG_0]     = 14,
-        u8 *end;
+        /* Function parameters */
-        u8 *base_ip;
+        [BPF_REG_1]     = 2,
-        u8 *ret0_ip;
+        [BPF_REG_2]     = 3,
-        u8 *exit_ip;
+        [BPF_REG_3]     = 4,
-        unsigned int off_load_word;
+        [BPF_REG_4]     = 5,
-        unsigned int off_load_half;
+        [BPF_REG_5]     = 6,
-        unsigned int off_load_byte;
+        /* Call saved registers */
-        unsigned int off_load_bmsh;
+        [BPF_REG_6]     = 7,
-        unsigned int off_load_iword;
+        [BPF_REG_7]     = 8,
-        unsigned int off_load_ihalf;
+        [BPF_REG_8]     = 9,
-        unsigned int off_load_ibyte;
+        [BPF_REG_9]     = 10,
+        /* BPF stack pointer */
+        [BPF_REG_FP]    = 13,
+        /* SKB data pointer */
+        [REG_SKB_DATA]  = 12,
+        /* Work registers for s390x backend */
+        [REG_W0]        = 0,
+        [REG_W1]        = 1,
+        [REG_L]         = 11,
+        [REG_15]        = 15,
 };
-#define BPF_SIZE_MAX    4096    /* Max size for program */
+static inline u32 reg(u32 dst_reg, u32 src_reg)
+{
+        return reg2hex[dst_reg] << 4 | reg2hex[src_reg];
+}
+static inline u32 reg_high(u32 reg)
+{
+        return reg2hex[reg] << 4;
+}
+static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
+{
+        u32 r1 = reg2hex[b1];
+        if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15)
+                jit->seen_reg[r1] = 1;
+}
+#define REG_SET_SEEN(b1)                                        \
+({                                                              \
+        reg_set_seen(jit, b1);                                  \
+})
+#define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
+/*
+ * EMIT macros for code generation
+ */
+#define _EMIT2(op)                                              \
+({                                                              \
+        if (jit->prg_buf)                                       \
+                *(u16 *) (jit->prg_buf + jit->prg) = op;        \
+        jit->prg += 2;                                          \
+})
-#define SEEN_DATAREF    1       /* might call external helpers */
+#define EMIT2(op, b1, b2)                                       \
-#define SEEN_XREG       2       /* ebx is used */
+({                                                              \
-#define SEEN_MEM        4       /* use mem[] for temporary storage */
+        _EMIT2(op | reg(b1, b2));                               \
-#define SEEN_RET0       8       /* pc_ret0 points to a valid return 0 */
+        REG_SET_SEEN(b1);                                       \
-#define SEEN_LITERAL    16      /* code uses literals */
+        REG_SET_SEEN(b2);                                       \
-#define SEEN_LOAD_WORD  32      /* code uses sk_load_word */
-#define SEEN_LOAD_HALF  64      /* code uses sk_load_half */
-#define SEEN_LOAD_BYTE  128     /* code uses sk_load_byte */
-#define SEEN_LOAD_BMSH  256     /* code uses sk_load_byte_msh */
-#define SEEN_LOAD_IWORD 512     /* code uses sk_load_word_ind */
-#define SEEN_LOAD_IHALF 1024    /* code uses sk_load_half_ind */
-#define SEEN_LOAD_IBYTE 2048    /* code uses sk_load_byte_ind */
-#define EMIT2(op)                                       \
-({                                                      \
-        if (jit->prg + 2 <= jit->mid)                   \
-                *(u16 *) jit->prg = op;                 \
-        jit->prg += 2;                                  \
 })
-#define EMIT4(op)                                       \
+#define _EMIT4(op)                                              \
-({                                                      \
+({                                                              \
-        if (jit->prg + 4 <= jit->mid)                   \
+        if (jit->prg_buf)                                       \
-                *(u32 *) jit->prg = op;                 \
+                *(u32 *) (jit->prg_buf + jit->prg) = op;        \
-        jit->prg += 4;                                  \
+        jit->prg += 4;                                          \
 })
-#define EMIT4_DISP(op, disp)                            \
+#define EMIT4(op, b1, b2)                                       \
-({                                                      \
+({                                                              \
-        unsigned int __disp = (disp) & 0xfff;           \
+        _EMIT4(op | reg(b1, b2));                               \
-        EMIT4(op | __disp);                             \
+        REG_SET_SEEN(b1);                                       \
+        REG_SET_SEEN(b2);                                       \
 })
-#define EMIT4_IMM(op, imm)                              \
+#define EMIT4_RRF(op, b1, b2, b3)                               \
-({                                                      \
+({                                                              \
-        unsigned int __imm = (imm) & 0xffff;            \
+        _EMIT4(op | reg_high(b3) << 8 | reg(b1, b2));           \
-        EMIT4(op | __imm);                              \
+        REG_SET_SEEN(b1);                                       \
+        REG_SET_SEEN(b2);                                       \
+        REG_SET_SEEN(b3);                                       \
 })
-#define EMIT4_PCREL(op, pcrel)                          \
+#define _EMIT4_DISP(op, disp)                                   \
-({                                                      \
+({                                                              \
-        long __pcrel = ((pcrel) >> 1) & 0xffff;         \
+        unsigned int __disp = (disp) & 0xfff;                   \
-        EMIT4(op | __pcrel);                            \
+        _EMIT4(op | __disp);                                    \
 })
-#define EMIT6(op1, op2)                                 \
+#define EMIT4_DISP(op, b1, b2, disp)                            \
-({                                                      \
+({                                                              \
-        if (jit->prg + 6 <= jit->mid) {                 \
+        _EMIT4_DISP(op | reg_high(b1) << 16 |                   \
-                *(u32 *) jit->prg = op1;                \
+                    reg_high(b2) << 8, disp);                   \
-                *(u16 *) (jit->prg + 4) = op2;          \
+        REG_SET_SEEN(b1);                                       \
-        }                                               \
+        REG_SET_SEEN(b2);                                       \
-        jit->prg += 6;                                  \
 })
-#define EMIT6_DISP(op1, op2, disp)                      \
+#define EMIT4_IMM(op, b1, imm)                                  \
-({                                                      \
+({                                                              \
-        unsigned int __disp = (disp) & 0xfff;           \
+        unsigned int __imm = (imm) & 0xffff;                    \
-        EMIT6(op1 | __disp, op2);                       \
+        _EMIT4(op | reg_high(b1) << 16 | __imm);                \
+        REG_SET_SEEN(b1);                                       \
 })
-#define EMIT6_IMM(op, imm)                              \
+#define EMIT4_PCREL(op, pcrel)                                  \
-({                                                      \
+({                                                              \
-        unsigned int __imm = (imm);                     \
+        long __pcrel = ((pcrel) >> 1) & 0xffff;                 \
-        EMIT6(op | (__imm >> 16), __imm & 0xffff);      \
+        _EMIT4(op | __pcrel);                                   \
 })
-#define EMIT_CONST(val)                                 \
+#define _EMIT6(op1, op2)                                        \
-({                                                      \
+({                                                              \
-        unsigned int ret;                               \
+        if (jit->prg_buf) {                                     \
-        ret = (unsigned int) (jit->lit - jit->base_ip); \
+                *(u32 *) (jit->prg_buf + jit->prg) = op1;       \
-        jit->seen |= SEEN_LITERAL;                      \
+                *(u16 *) (jit->prg_buf + jit->prg + 4) = op2;   \
-        if (jit->lit + 4 <= jit->end)                   \
+        }                                                       \
-                *(u32 *) jit->lit = val;                \
+        jit->prg += 6;                                          \
-        jit->lit += 4;                                  \
-        ret;                                            \
 })
-#define EMIT_FN_CONST(bit, fn)                          \
+#define _EMIT6_DISP(op1, op2, disp)                             \
-({                                                      \
+({                                                              \
-        unsigned int ret;                               \
+        unsigned int __disp = (disp) & 0xfff;                   \
-        ret = (unsigned int) (jit->lit - jit->base_ip); \
+        _EMIT6(op1 | __disp, op2);                              \
-        if (jit->seen & bit) {                          \
-                jit->seen |= SEEN_LITERAL;              \
-                if (jit->lit + 8 <= jit->end)           \
-                        *(void **) jit->lit = fn;       \
-                jit->lit += 8;                          \
-        }                                               \
-        ret;                                            \
 })
-static void bpf_jit_fill_hole(void *area, unsigned int size)
+#define EMIT6_DISP(op1, op2, b1, b2, b3, disp)                  \
+({                                                              \
+        _EMIT6_DISP(op1 | reg(b1, b2) << 16 |                   \
+                    reg_high(b3) << 8, op2, disp);              \
+        REG_SET_SEEN(b1);                                       \
+        REG_SET_SEEN(b2);                                       \
+        REG_SET_SEEN(b3);                                       \
+})
+#define _EMIT6_DISP_LH(op1, op2, disp)                          \
+({                                                              \
+        unsigned int __disp_h = ((u32)disp) & 0xff000;          \
+        unsigned int __disp_l = ((u32)disp) & 0x00fff;          \
+        _EMIT6(op1 | __disp_l, op2 | __disp_h >> 4);            \
+})
+#define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp)               \
+({                                                              \
+        _EMIT6_DISP_LH(op1 | reg(b1, b2) << 16 |                \
+                       reg_high(b3) << 8, op2, disp);           \
+        REG_SET_SEEN(b1);                                       \
+        REG_SET_SEEN(b2);                                       \
+        REG_SET_SEEN(b3);                                       \
+})
+#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask)             \
+({                                                              \
+        /* Branch instruction needs 6 bytes */                  \
+        int rel = (addrs[i + off + 1] - (addrs[i + 1] - 6)) / 2;\
+        _EMIT6(op1 | reg(b1, b2) << 16 | rel, op2 | mask);      \
+        REG_SET_SEEN(b1);                                       \
+        REG_SET_SEEN(b2);                                       \
+})
+#define _EMIT6_IMM(op, imm)                                     \
+({                                                              \
+        unsigned int __imm = (imm);                             \
+        _EMIT6(op | (__imm >> 16), __imm & 0xffff);             \
+})
+#define EMIT6_IMM(op, b1, imm)                                  \
+({                                                              \
+        _EMIT6_IMM(op | reg_high(b1) << 16, imm);               \
+        REG_SET_SEEN(b1);                                       \
+})
+#define EMIT_CONST_U32(val)                                     \
+({                                                              \
+        unsigned int ret;                                       \
+        ret = jit->lit - jit->base_ip;                          \
+        jit->seen |= SEEN_LITERAL;                              \
+        if (jit->prg_buf)                                       \
+                *(u32 *) (jit->prg_buf + jit->lit) = (u32) val; \
+        jit->lit += 4;                                          \
+        ret;                                                    \
+})
+#define EMIT_CONST_U64(val)                                     \
+({                                                              \
+        unsigned int ret;                                       \
+        ret = jit->lit - jit->base_ip;                          \
+        jit->seen |= SEEN_LITERAL;                              \
+        if (jit->prg_buf)                                       \
+                *(u64 *) (jit->prg_buf + jit->lit) = (u64) val; \
+        jit->lit += 8;                                          \
+        ret;                                                    \
+})
+#define EMIT_ZERO(b1)                                           \
+({                                                              \
+        /* llgfr %dst,%dst (zero extend to 64 bit) */           \
+        EMIT4(0xb9160000, b1, b1);                              \
+        REG_SET_SEEN(b1);                                       \
+})
+/*
+ * Fill whole space with illegal instructions
+ */
+static void jit_fill_hole(void *area, unsigned int size)
 {
-        /* Fill whole space with illegal instructions */
        memset(area, 0, size);
 }
-static void bpf_jit_prologue(struct bpf_jit *jit)
+/*
+ * Save registers from "rs" (register start) to "re" (register end) on stack
+ */
+static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
+{
+        u32 off = 72 + (rs - 6) * 8;
+        if (rs == re)
+                /* stg %rs,off(%r15) */
+                _EMIT6(0xe300f000 | rs << 20 | off, 0x0024);
+        else
+                /* stmg %rs,%re,off(%r15) */
+                _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off);
+}
+/*
+ * Restore registers from "rs" (register start) to "re" (register end) on stack
+ */
+static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)
 {
-        /* Save registers and create stack frame if necessary */
+        u32 off = 72 + (rs - 6) * 8;
-        if (jit->seen & SEEN_DATAREF) {
-                /* stmg %r8,%r15,88(%r15) */
+        if (jit->seen & SEEN_STACK)
-                EMIT6(0xeb8ff058, 0x0024);
+                off += STK_OFF;
-                /* lgr %r14,%r15 */
-                EMIT4(0xb90400ef);
+        if (rs == re)
-                /* aghi %r15,<offset> */
+                /* lg %rs,off(%r15) */
-                EMIT4_IMM(0xa7fb0000, (jit->seen & SEEN_MEM) ? -112 : -80);
+                _EMIT6(0xe300f000 | rs << 20 | off, 0x0004);
-                /* stg %r14,152(%r15) */
+        else
-                EMIT6(0xe3e0f098, 0x0024);
+                /* lmg %rs,%re,off(%r15) */
-        } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL))
+                _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off);
-                /* stmg %r12,%r13,120(%r15) */
+}
-                EMIT6(0xebcdf078, 0x0024);
-        else if (jit->seen & SEEN_XREG)
-                /* stg %r12,120(%r15) */
-                EMIT6(0xe3c0f078, 0x0024);
-        else if (jit->seen & SEEN_LITERAL)
-                /* stg %r13,128(%r15) */
-                EMIT6(0xe3d0f080, 0x0024);
+/*
+ * Return first seen register (from start)
+ */
+static int get_start(struct bpf_jit *jit, int start)
+{
+        int i;
+        for (i = start; i <= 15; i++) {
+                if (jit->seen_reg[i])
+                        return i;
+        }
+        return 0;
+}
+/*
+ * Return last seen register (from start) (gap >= 2)
+ */
+static int get_end(struct bpf_jit *jit, int start)
+{
+        int i;
+        for (i = start; i < 15; i++) {
+                if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])
+                        return i - 1;
+        }
+        return jit->seen_reg[15] ? 15 : 14;
+}
+#define REGS_SAVE       1
+#define REGS_RESTORE    0
+/*
+ * Save and restore clobbered registers (6-15) on stack.
+ * We save/restore registers in chunks with gap >= 2 registers.
+ */
+static void save_restore_regs(struct bpf_jit *jit, int op)
+{
+        int re = 6, rs;
+        do {
+                rs = get_start(jit, re);
+                if (!rs)
+                        break;
+                re = get_end(jit, rs + 1);
+                if (op == REGS_SAVE)
+                        save_regs(jit, rs, re);
+                else
+                        restore_regs(jit, rs, re);
+                re++;
+        } while (re <= 15);
+}
+/*
+ * Emit function prologue
+ *
+ * Save registers and create stack frame if necessary.
+ * See stack frame layout desription in "bpf_jit.h"!
+ */
+static void bpf_jit_prologue(struct bpf_jit *jit)
+{
+        /* Save registers */
+        save_restore_regs(jit, REGS_SAVE);
        /* Setup literal pool */
        if (jit->seen & SEEN_LITERAL) {
                /* basr %r13,0 */
-                EMIT2(0x0dd0);
+                EMIT2(0x0d00, REG_L, REG_0);
                jit->base_ip = jit->prg;
        }
-        jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word);
+        /* Setup stack and backchain */
-        jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half);
+        if (jit->seen & SEEN_STACK) {
-        jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte);
+                /* lgr %bfp,%r15 (BPF frame pointer) */
-        jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh);
+                EMIT4(0xb9040000, BPF_REG_FP, REG_15);
-        jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind);
+                /* aghi %r15,-STK_OFF */
-        jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind);
+                EMIT4_IMM(0xa70b0000, REG_15, -STK_OFF);
-        jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind);
+                if (jit->seen & SEEN_FUNC)
+                        /* stg %bfp,152(%r15) (backchain) */
-        /* Filter needs to access skb data */
+                        EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_FP, REG_0,
-        if (jit->seen & SEEN_DATAREF) {
+                                      REG_15, 152);
-                /* l %r11,<len>(%r2) */
+        }
-                EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len));
+        /*
-                /* s %r11,<data_len>(%r2) */
+         * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
-                EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len));
+         * we store the SKB header length on the stack and the SKB data
-                /* lg %r10,<data>(%r2) */
+         * pointer in REG_SKB_DATA.
-                EMIT6_DISP(0xe3a02000, 0x0004,
+         */
-                           offsetof(struct sk_buff, data));
+        if (jit->seen & SEEN_SKB) {
+                /* Header length: llgf %w1,<len>(%b1) */
+                EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,
+                              offsetof(struct sk_buff, len));
+                /* s %w1,<data_len>(%b1) */
+                EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,
+                           offsetof(struct sk_buff, data_len));
+                /* stg %w1,ST_OFF_HLEN(%r0,%r15) */
+                EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15,
+                              STK_OFF_HLEN);
+                /* lg %skb_data,data_off(%b1) */
+                EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
+                              BPF_REG_1, offsetof(struct sk_buff, data));
        }
+        /* BPF compatibility: clear A (%b7) and X (%b8) registers */
+        if (REG_SEEN(BPF_REG_7))
+                /* lghi %b7,0 */
+                EMIT4_IMM(0xa7090000, BPF_REG_7, 0);
+        if (REG_SEEN(BPF_REG_8))
+                /* lghi %b8,0 */
+                EMIT4_IMM(0xa7090000, BPF_REG_8, 0);
 }
+/*
+ * Function epilogue
+ */
 static void bpf_jit_epilogue(struct bpf_jit *jit)
 {
        /* Return 0 */
        if (jit->seen & SEEN_RET0) {
                jit->ret0_ip = jit->prg;
-                /* lghi %r2,0 */
+                /* lghi %b0,0 */
-                EMIT4(0xa7290000);
+                EMIT4_IMM(0xa7090000, BPF_REG_0, 0);
        }
        jit->exit_ip = jit->prg;
+        /* Load exit code: lgr %r2,%b0 */
+        EMIT4(0xb9040000, REG_2, BPF_REG_0);
        /* Restore registers */
-        if (jit->seen & SEEN_DATAREF)
+        save_restore_regs(jit, REGS_RESTORE);
-                /* lmg %r8,%r15,<offset>(%r15) */
-                EMIT6_DISP(0xeb8ff000, 0x0004,
-                           (jit->seen & SEEN_MEM) ? 200 : 168);
-        else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL))
-                /* lmg %r12,%r13,120(%r15) */
-                EMIT6(0xebcdf078, 0x0004);
-        else if (jit->seen & SEEN_XREG)
-                /* lg %r12,120(%r15) */
-                EMIT6(0xe3c0f078, 0x0004);
-        else if (jit->seen & SEEN_LITERAL)
-                /* lg %r13,128(%r15) */
-                EMIT6(0xe3d0f080, 0x0004);
        /* br %r14 */
-        EMIT2(0x07fe);
+        _EMIT2(0x07fe);
 }
 /*
- * make sure we dont leak kernel information to user
+ * Compile one eBPF instruction into s390x code
 */
-static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter)
+static int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i)
 {
-        /* Clear temporary memory if (seen & SEEN_MEM) */
+        struct bpf_insn *insn = &fp->insnsi[i];
-        if (jit->seen & SEEN_MEM)
+        int jmp_off, last, insn_count = 1;
-                /* xc 0(64,%r15),0(%r15) */
+        unsigned int func_addr, mask;
-                EMIT6(0xd73ff000, 0xf000);
+        u32 dst_reg = insn->dst_reg;
-        /* Clear X if (seen & SEEN_XREG) */
+        u32 src_reg = insn->src_reg;
-        if (jit->seen & SEEN_XREG)
+        u32 *addrs = jit->addrs;
-                /* lhi %r12,0 */
+        s32 imm = insn->imm;
-                EMIT4(0xa7c80000);
+        s16 off = insn->off;
-        /* Clear A if the first register does not set it. */
-        switch (filter[0].code) {
-        case BPF_LD | BPF_W | BPF_ABS:
-        case BPF_LD | BPF_H | BPF_ABS:
-        case BPF_LD | BPF_B | BPF_ABS:
-        case BPF_LD | BPF_W | BPF_LEN:
-        case BPF_LD | BPF_W | BPF_IND:
-        case BPF_LD | BPF_H | BPF_IND:
-        case BPF_LD | BPF_B | BPF_IND:
-        case BPF_LD | BPF_IMM:
-        case BPF_LD | BPF_MEM:
-        case BPF_MISC | BPF_TXA:
-        case BPF_RET | BPF_K:
-                /* first instruction sets A register */
-                break;
-        default: /* A = 0 */
-                /* lhi %r5,0 */
-                EMIT4(0xa7580000);
-        }
-}
-static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter,
+        switch (insn->code) {
-                        unsigned int *addrs, int i, int last)
+        /*
-{
+         * BPF_MOV
-        unsigned int K;
+         */
-        int offset;
+        case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */
-        unsigned int mask;
+                /* llgfr %dst,%src */
-        u16 code;
+                EMIT4(0xb9160000, dst_reg, src_reg);
+                break;
-        K = filter->k;
+        case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
-        code = bpf_anc_helper(filter);
+                /* lgr %dst,%src */
+                EMIT4(0xb9040000, dst_reg, src_reg);
-        switch (code) {
+                break;
-        case BPF_ALU | BPF_ADD | BPF_X: /* A += X */
+        case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */
-                jit->seen |= SEEN_XREG;
+                /* llilf %dst,imm */
-                /* ar %r5,%r12 */
+                EMIT6_IMM(0xc00f0000, dst_reg, imm);
-                EMIT2(0x1a5c);
+                break;
-                break;
+        case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */
-        case BPF_ALU | BPF_ADD | BPF_K: /* A += K */
+                /* lgfi %dst,imm */
-                if (!K)
+                EMIT6_IMM(0xc0010000, dst_reg, imm);
+                break;
+        /*
+         * BPF_LD 64
+         */
+        case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
+        {
+                /* 16 byte instruction that uses two 'struct bpf_insn' */
+                u64 imm64;
+                imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32;
+                /* lg %dst,<d(imm)>(%l) */
+                EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, REG_0, REG_L,
+                              EMIT_CONST_U64(imm64));
+                insn_count = 2;
+                break;
+        }
+        /*
+         * BPF_ADD
+         */
+        case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */
+                /* ar %dst,%src */
+                EMIT2(0x1a00, dst_reg, src_reg);
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */
+                /* agr %dst,%src */
+                EMIT4(0xb9080000, dst_reg, src_reg);
+                break;
+        case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */
+                if (!imm)
                        break;
-                if (K <= 16383)
+                /* alfi %dst,imm */
-                        /* ahi %r5,<K> */
+                EMIT6_IMM(0xc20b0000, dst_reg, imm);
-                        EMIT4_IMM(0xa75a0000, K);
+                EMIT_ZERO(dst_reg);
-                else if (test_facility(21))
+                break;
-                        /* alfi %r5,<K> */
+        case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */
-                        EMIT6_IMM(0xc25b0000, K);
+                if (!imm)
-                else
+                        break;
-                        /* a %r5,<d(K)>(%r13) */
+                /* agfi %dst,imm */
-                        EMIT4_DISP(0x5a50d000, EMIT_CONST(K));
+                EMIT6_IMM(0xc2080000, dst_reg, imm);
+                break;
+        /*
+         * BPF_SUB
+         */
+        case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */
+                /* sr %dst,%src */
+                EMIT2(0x1b00, dst_reg, src_reg);
+                EMIT_ZERO(dst_reg);
                break;
-        case BPF_ALU | BPF_SUB | BPF_X: /* A -= X */
+        case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */
-                jit->seen |= SEEN_XREG;
+                /* sgr %dst,%src */
-                /* sr %r5,%r12 */
+                EMIT4(0xb9090000, dst_reg, src_reg);
-                EMIT2(0x1b5c);
                break;
-        case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */
+        case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */
-                if (!K)
+                if (!imm)
                        break;
-                if (K <= 16384)
+                /* alfi %dst,-imm */
-                        /* ahi %r5,-K */
+                EMIT6_IMM(0xc20b0000, dst_reg, -imm);
-                        EMIT4_IMM(0xa75a0000, -K);
+                EMIT_ZERO(dst_reg);
-                else if (test_facility(21))
-                        /* alfi %r5,-K */
-                        EMIT6_IMM(0xc25b0000, -K);
-                else
-                        /* s %r5,<d(K)>(%r13) */
-                        EMIT4_DISP(0x5b50d000, EMIT_CONST(K));
-                break;
-        case BPF_ALU | BPF_MUL | BPF_X: /* A *= X */
-                jit->seen |= SEEN_XREG;
-                /* msr %r5,%r12 */
-                EMIT4(0xb252005c);
-                break;
-        case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */
-                if (K <= 16383)
-                        /* mhi %r5,K */
-                        EMIT4_IMM(0xa75c0000, K);
-                else if (test_facility(34))
-                        /* msfi %r5,<K> */
-                        EMIT6_IMM(0xc2510000, K);
-                else
-                        /* ms %r5,<d(K)>(%r13) */
-                        EMIT4_DISP(0x7150d000, EMIT_CONST(K));
                break;
-        case BPF_ALU | BPF_DIV | BPF_X: /* A /= X */
+        case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */
-                jit->seen |= SEEN_XREG | SEEN_RET0;
+                if (!imm)
-                /* ltr %r12,%r12 */
+                        break;
-                EMIT2(0x12cc);
+                /* agfi %dst,-imm */
-                /* jz <ret0> */
+                EMIT6_IMM(0xc2080000, dst_reg, -imm);
-                EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg));
+                break;
-                /* lhi %r4,0 */
+        /*
-                EMIT4(0xa7480000);
+         * BPF_MUL
-                /* dlr %r4,%r12 */
+         */
-                EMIT4(0xb997004c);
+        case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */
-                break;
+                /* msr %dst,%src */
-        case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */
+                EMIT4(0xb2520000, dst_reg, src_reg);
-                if (K == 1)
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */
+                /* msgr %dst,%src */
+                EMIT4(0xb90c0000, dst_reg, src_reg);
+                break;
+        case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */
+                if (imm == 1)
+                        break;
+                /* msfi %r5,imm */
+                EMIT6_IMM(0xc2010000, dst_reg, imm);
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */
+                if (imm == 1)
                        break;
-                /* lhi %r4,0 */
+                /* msgfi %dst,imm */
-                EMIT4(0xa7480000);
+                EMIT6_IMM(0xc2000000, dst_reg, imm);
-                /* dl %r4,<d(K)>(%r13) */
+                break;
-                EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K));
+        /*
-                break;
+         * BPF_DIV / BPF_MOD
-        case BPF_ALU | BPF_MOD | BPF_X: /* A %= X */
+         */
-                jit->seen |= SEEN_XREG | SEEN_RET0;
+        case BPF_ALU | BPF_DIV | BPF_X: /* dst = (u32) dst / (u32) src */
-                /* ltr %r12,%r12 */
+        case BPF_ALU | BPF_MOD | BPF_X: /* dst = (u32) dst % (u32) src */
-                EMIT2(0x12cc);
+        {
+                int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
+                jit->seen |= SEEN_RET0;
+                /* ltr %src,%src (if src == 0 goto fail) */
+                EMIT2(0x1200, src_reg, src_reg);
+                /* jz <ret0> */
+                EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
+                /* lhi %w0,0 */
+                EMIT4_IMM(0xa7080000, REG_W0, 0);
+                /* lr %w1,%dst */
+                EMIT2(0x1800, REG_W1, dst_reg);
+                /* dlr %w0,%src */
+                EMIT4(0xb9970000, REG_W0, src_reg);
+                /* llgfr %dst,%rc */
+                EMIT4(0xb9160000, dst_reg, rc_reg);
+                break;
+        }
+        case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / (u32) src */
+        case BPF_ALU64 | BPF_MOD | BPF_X: /* dst = dst % (u32) src */
+        {
+                int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
+                jit->seen |= SEEN_RET0;
+                /* ltgr %src,%src (if src == 0 goto fail) */
+                EMIT4(0xb9020000, src_reg, src_reg);
                /* jz <ret0> */
-                EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg));
+                EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
-                /* lhi %r4,0 */
+                /* lghi %w0,0 */
-                EMIT4(0xa7480000);
+                EMIT4_IMM(0xa7090000, REG_W0, 0);
-                /* dlr %r4,%r12 */
+                /* lgr %w1,%dst */
-                EMIT4(0xb997004c);
+                EMIT4(0xb9040000, REG_W1, dst_reg);
-                /* lr %r5,%r4 */
+                /* llgfr %dst,%src (u32 cast) */
-                EMIT2(0x1854);
+                EMIT4(0xb9160000, dst_reg, src_reg);
-                break;
+                /* dlgr %w0,%dst */
-        case BPF_ALU | BPF_MOD | BPF_K: /* A %= K */
+                EMIT4(0xb9870000, REG_W0, dst_reg);
-                if (K == 1) {
+                /* lgr %dst,%rc */
-                        /* lhi %r5,0 */
+                EMIT4(0xb9040000, dst_reg, rc_reg);
-                        EMIT4(0xa7580000);
+                break;
+        }
+        case BPF_ALU | BPF_DIV | BPF_K: /* dst = (u32) dst / (u32) imm */
+        case BPF_ALU | BPF_MOD | BPF_K: /* dst = (u32) dst % (u32) imm */
+        {
+                int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
+                if (imm == 1) {
+                        if (BPF_OP(insn->code) == BPF_MOD)
+                                /* lhgi %dst,0 */
+                                EMIT4_IMM(0xa7090000, dst_reg, 0);
                        break;
                }
-                /* lhi %r4,0 */
+                /* lhi %w0,0 */
-                EMIT4(0xa7480000);
+                EMIT4_IMM(0xa7080000, REG_W0, 0);
-                /* dl %r4,<d(K)>(%r13) */
+                /* lr %w1,%dst */
-                EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K));
+                EMIT2(0x1800, REG_W1, dst_reg);
-                /* lr %r5,%r4 */
+                /* dl %w0,<d(imm)>(%l) */
-                EMIT2(0x1854);
+                EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
-                break;
+                              EMIT_CONST_U32(imm));
-        case BPF_ALU | BPF_AND | BPF_X: /* A &= X */
+                /* llgfr %dst,%rc */
-                jit->seen |= SEEN_XREG;
+                EMIT4(0xb9160000, dst_reg, rc_reg);
-                /* nr %r5,%r12 */
+                break;
-                EMIT2(0x145c);
+        }
-                break;
+        case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / (u32) imm */
-        case BPF_ALU | BPF_AND | BPF_K: /* A &= K */
+        case BPF_ALU64 | BPF_MOD | BPF_K: /* dst = dst % (u32) imm */
-                if (test_facility(21))
+        {
-                        /* nilf %r5,<K> */
+                int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
-                        EMIT6_IMM(0xc05b0000, K);
-                else
+                if (imm == 1) {
-                        /* n %r5,<d(K)>(%r13) */
+                        if (BPF_OP(insn->code) == BPF_MOD)
-                        EMIT4_DISP(0x5450d000, EMIT_CONST(K));
+                                /* lhgi %dst,0 */
-                break;
+                                EMIT4_IMM(0xa7090000, dst_reg, 0);
-        case BPF_ALU | BPF_OR | BPF_X: /* A |= X */
+                        break;
-                jit->seen |= SEEN_XREG;
+                }
-                /* or %r5,%r12 */
+                /* lghi %w0,0 */
-                EMIT2(0x165c);
+                EMIT4_IMM(0xa7090000, REG_W0, 0);
-                break;
+                /* lgr %w1,%dst */
-        case BPF_ALU | BPF_OR | BPF_K: /* A |= K */
+                EMIT4(0xb9040000, REG_W1, dst_reg);
-                if (test_facility(21))
+                /* dlg %w0,<d(imm)>(%l) */
-                        /* oilf %r5,<K> */
+                EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
-                        EMIT6_IMM(0xc05d0000, K);
+                              EMIT_CONST_U64((u32) imm));
-                else
+                /* lgr %dst,%rc */
-                        /* o %r5,<d(K)>(%r13) */
+                EMIT4(0xb9040000, dst_reg, rc_reg);
-                        EMIT4_DISP(0x5650d000, EMIT_CONST(K));
+                break;
+        }
+        /*
+         * BPF_AND
+         */
+        case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */
+                /* nr %dst,%src */
+                EMIT2(0x1400, dst_reg, src_reg);
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
+                /* ngr %dst,%src */
+                EMIT4(0xb9800000, dst_reg, src_reg);
+                break;
+        case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */
+                /* nilf %dst,imm */
+                EMIT6_IMM(0xc00b0000, dst_reg, imm);
+                EMIT_ZERO(dst_reg);
                break;
-        case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */
+        case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
-        case BPF_ALU | BPF_XOR | BPF_X:
+                /* ng %dst,<d(imm)>(%l) */
-                jit->seen |= SEEN_XREG;
+                EMIT6_DISP_LH(0xe3000000, 0x0080, dst_reg, REG_0, REG_L,
-                /* xr %r5,%r12 */
+                              EMIT_CONST_U64(imm));
-                EMIT2(0x175c);
                break;
-        case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */
+        /*
-                if (!K)
+         * BPF_OR
+         */
+        case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
+                /* or %dst,%src */
+                EMIT2(0x1600, dst_reg, src_reg);
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
+                /* ogr %dst,%src */
+                EMIT4(0xb9810000, dst_reg, src_reg);
+                break;
+        case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */
+                /* oilf %dst,imm */
+                EMIT6_IMM(0xc00d0000, dst_reg, imm);
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */
+                /* og %dst,<d(imm)>(%l) */
+                EMIT6_DISP_LH(0xe3000000, 0x0081, dst_reg, REG_0, REG_L,
+                              EMIT_CONST_U64(imm));
+                break;
+        /*
+         * BPF_XOR
+         */
+        case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */
+                /* xr %dst,%src */
+                EMIT2(0x1700, dst_reg, src_reg);
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */
+                /* xgr %dst,%src */
+                EMIT4(0xb9820000, dst_reg, src_reg);
+                break;
+        case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */
+                if (!imm)
                        break;
-                /* x %r5,<d(K)>(%r13) */
+                /* xilf %dst,imm */
-                EMIT4_DISP(0x5750d000, EMIT_CONST(K));
+                EMIT6_IMM(0xc0070000, dst_reg, imm);
+                EMIT_ZERO(dst_reg);
+                break;
+        case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */
+                /* xg %dst,<d(imm)>(%l) */
+                EMIT6_DISP_LH(0xe3000000, 0x0082, dst_reg, REG_0, REG_L,
+                              EMIT_CONST_U64(imm));
+                break;
+        /*
+         * BPF_LSH
+         */
+        case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */
+                /* sll %dst,0(%src) */
+                EMIT4_DISP(0x89000000, dst_reg, src_reg, 0);
+                EMIT_ZERO(dst_reg);
                break;
-        case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */
+        case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */
-                jit->seen |= SEEN_XREG;
+                /* sllg %dst,%dst,0(%src) */
-                /* sll %r5,0(%r12) */
+                EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0);
-                EMIT4(0x8950c000);
                break;
-        case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */
+        case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */
-                if (K == 0)
+                if (imm == 0)
                        break;
-                /* sll %r5,K */
+                /* sll %dst,imm(%r0) */
-                EMIT4_DISP(0x89500000, K);
+                EMIT4_DISP(0x89000000, dst_reg, REG_0, imm);
+                EMIT_ZERO(dst_reg);
                break;
-        case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */
+        case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */
-                jit->seen |= SEEN_XREG;
+                if (imm == 0)
-                /* srl %r5,0(%r12) */
+                        break;
-                EMIT4(0x8850c000);
+                /* sllg %dst,%dst,imm(%r0) */
+                EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm);
+                break;
+        /*
+         * BPF_RSH
+         */
+        case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */
+                /* srl %dst,0(%src) */
+                EMIT4_DISP(0x88000000, dst_reg, src_reg, 0);
+                EMIT_ZERO(dst_reg);
                break;
-        case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */
+        case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */
-                if (K == 0)
+                /* srlg %dst,%dst,0(%src) */
+                EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0);
+                break;
+        case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */
+                if (imm == 0)
                        break;
-                /* srl %r5,K */
+                /* srl %dst,imm(%r0) */
-                EMIT4_DISP(0x88500000, K);
+                EMIT4_DISP(0x88000000, dst_reg, REG_0, imm);
-                break;
+                EMIT_ZERO(dst_reg);
-        case BPF_ALU | BPF_NEG: /* A = -A */
+                break;
-                /* lcr %r5,%r5 */
+        case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */
-                EMIT2(0x1355);
+                if (imm == 0)
-                break;
-        case BPF_JMP | BPF_JA: /* ip += K */
-                offset = addrs[i + K] + jit->start - jit->prg;
-                EMIT4_PCREL(0xa7f40000, offset);
-                break;
-        case BPF_JMP | BPF_JGT | BPF_K: /* ip += (A > K) ? jt : jf */
-                mask = 0x200000; /* jh */
-                goto kbranch;
-        case BPF_JMP | BPF_JGE | BPF_K: /* ip += (A >= K) ? jt : jf */
-                mask = 0xa00000; /* jhe */
-                goto kbranch;
-        case BPF_JMP | BPF_JEQ | BPF_K: /* ip += (A == K) ? jt : jf */
-                mask = 0x800000; /* je */
-kbranch:        /* Emit compare if the branch targets are different */
-                if (filter->jt != filter->jf) {
-                        if (test_facility(21))
-                                /* clfi %r5,<K> */
-                                EMIT6_IMM(0xc25f0000, K);
-                        else
-                                /* cl %r5,<d(K)>(%r13) */
-                                EMIT4_DISP(0x5550d000, EMIT_CONST(K));
-                }
-branch:         if (filter->jt == filter->jf) {
-                        if (filter->jt == 0)
-                                break;
-                        /* j <jt> */
-                        offset = addrs[i + filter->jt] + jit->start - jit->prg;
-                        EMIT4_PCREL(0xa7f40000, offset);
                        break;
-                }
+                /* srlg %dst,%dst,imm(%r0) */
-                if (filter->jt != 0) {
+                EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm);
-                        /* brc  <mask>,<jt> */
-                        offset = addrs[i + filter->jt] + jit->start - jit->prg;
-                        EMIT4_PCREL(0xa7040000 | mask, offset);
-                }
-                if (filter->jf != 0) {
-                        /* brc  <mask^15>,<jf> */
-                        offset = addrs[i + filter->jf] + jit->start - jit->prg;
-                        EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset);
-                }
                break;
-        case BPF_JMP | BPF_JSET | BPF_K: /* ip += (A & K) ? jt : jf */
+        /*
-                mask = 0x700000; /* jnz */
+         * BPF_ARSH
-                /* Emit test if the branch targets are different */
+         */
-                if (filter->jt != filter->jf) {
+        case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */
-                        if (K > 65535) {
+                /* srag %dst,%dst,0(%src) */
-                                /* lr %r4,%r5 */
+                EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
-                                EMIT2(0x1845);
+                break;
-                                /* n %r4,<d(K)>(%r13) */
+        case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */
-                                EMIT4_DISP(0x5440d000, EMIT_CONST(K));
+                if (imm == 0)
-                        } else
+                        break;
-                                /* tmll %r5,K */
+                /* srag %dst,%dst,imm(%r0) */
-                                EMIT4_IMM(0xa7510000, K);
+                EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm);
-                }
+                break;
-                goto branch;
+        /*
-        case BPF_JMP | BPF_JGT | BPF_X: /* ip += (A > X) ? jt : jf */
+         * BPF_NEG
-                mask = 0x200000; /* jh */
+         */
-                goto xbranch;
+        case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */
-        case BPF_JMP | BPF_JGE | BPF_X: /* ip += (A >= X) ? jt : jf */
+                /* lcr %dst,%dst */
-                mask = 0xa00000; /* jhe */
+                EMIT2(0x1300, dst_reg, dst_reg);
-                goto xbranch;
+                EMIT_ZERO(dst_reg);
-        case BPF_JMP | BPF_JEQ | BPF_X: /* ip += (A == X) ? jt : jf */
+                break;
-                mask = 0x800000; /* je */
+        case BPF_ALU64 | BPF_NEG: /* dst = -dst */
-xbranch:        /* Emit compare if the branch targets are different */
+                /* lcgr %dst,%dst */
-                if (filter->jt != filter->jf) {
+                EMIT4(0xb9130000, dst_reg, dst_reg);
-                        jit->seen |= SEEN_XREG;
+                break;
-                        /* clr %r5,%r12 */
+        /*
-                        EMIT2(0x155c);
+         * BPF_FROM_BE/LE
-                }
+         */
-                goto branch;
+        case BPF_ALU | BPF_END | BPF_FROM_BE:
-        case BPF_JMP | BPF_JSET | BPF_X: /* ip += (A & X) ? jt : jf */
+                /* s390 is big endian, therefore only clear high order bytes */
-                mask = 0x700000; /* jnz */
+                switch (imm) {
-                /* Emit test if the branch targets are different */
+                case 16: /* dst = (u16) cpu_to_be16(dst) */
-                if (filter->jt != filter->jf) {
+                        /* llghr %dst,%dst */
-                        jit->seen |= SEEN_XREG;
+                        EMIT4(0xb9850000, dst_reg, dst_reg);
-                        /* lr %r4,%r5 */
+                        break;
-                        EMIT2(0x1845);
+                case 32: /* dst = (u32) cpu_to_be32(dst) */
-                        /* nr %r4,%r12 */
+                        /* llgfr %dst,%dst */
-                        EMIT2(0x144c);
+                        EMIT4(0xb9160000, dst_reg, dst_reg);
+                        break;
+                case 64: /* dst = (u64) cpu_to_be64(dst) */
+                        break;
                }
-                goto branch;
-        case BPF_LD | BPF_W | BPF_ABS: /* A = *(u32 *) (skb->data+K) */
-                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD;
-                offset = jit->off_load_word;
-                goto load_abs;
-        case BPF_LD | BPF_H | BPF_ABS: /* A = *(u16 *) (skb->data+K) */
-                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF;
-                offset = jit->off_load_half;
-                goto load_abs;
-        case BPF_LD | BPF_B | BPF_ABS: /* A = *(u8 *) (skb->data+K) */
-                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE;
-                offset = jit->off_load_byte;
-load_abs:       if ((int) K < 0)
-                        goto out;
-call_fn:        /* lg %r1,<d(function)>(%r13) */
-                EMIT6_DISP(0xe310d000, 0x0004, offset);
-                /* l %r3,<d(K)>(%r13) */
-                EMIT4_DISP(0x5830d000, EMIT_CONST(K));
-                /* basr %r8,%r1 */
-                EMIT2(0x0d81);
-                /* jnz <ret0> */
-                EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg));
                break;
-        case BPF_LD | BPF_W | BPF_IND: /* A = *(u32 *) (skb->data+K+X) */
+        case BPF_ALU | BPF_END | BPF_FROM_LE:
-                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD;
+                switch (imm) {
-                offset = jit->off_load_iword;
+                case 16: /* dst = (u16) cpu_to_le16(dst) */
-                goto call_fn;
+                        /* lrvr %dst,%dst */
-        case BPF_LD | BPF_H | BPF_IND: /* A = *(u16 *) (skb->data+K+X) */
+                        EMIT4(0xb91f0000, dst_reg, dst_reg);
-                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF;
+                        /* srl %dst,16(%r0) */
-                offset = jit->off_load_ihalf;
+                        EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);
-                goto call_fn;
+                        /* llghr %dst,%dst */
-        case BPF_LD | BPF_B | BPF_IND: /* A = *(u8 *) (skb->data+K+X) */
+                        EMIT4(0xb9850000, dst_reg, dst_reg);
-                jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE;
+                        break;
-                offset = jit->off_load_ibyte;
+                case 32: /* dst = (u32) cpu_to_le32(dst) */
-                goto call_fn;
+                        /* lrvr %dst,%dst */
-        case BPF_LDX | BPF_B | BPF_MSH:
+                        EMIT4(0xb91f0000, dst_reg, dst_reg);
-                /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */
+                        /* llgfr %dst,%dst */
-                jit->seen |= SEEN_RET0;
+                        EMIT4(0xb9160000, dst_reg, dst_reg);
-                if ((int) K < 0) {
+                        break;
-                        /* j <ret0> */
+                case 64: /* dst = (u64) cpu_to_le64(dst) */
-                        EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg));
+                        /* lrvgr %dst,%dst */
+                        EMIT4(0xb90f0000, dst_reg, dst_reg);
                        break;
                }
-                jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH;
-                offset = jit->off_load_bmsh;
-                goto call_fn;
-        case BPF_LD | BPF_W | BPF_LEN: /*       A = skb->len; */
-                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
-                /* l %r5,<d(len)>(%r2) */
-                EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len));
-                break;
-        case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */
-                jit->seen |= SEEN_XREG;
-                /* l %r12,<d(len)>(%r2) */
-                EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len));
-                break;
-        case BPF_LD | BPF_IMM: /* A = K */
-                if (K <= 16383)
-                        /* lhi %r5,K */
-                        EMIT4_IMM(0xa7580000, K);
-                else if (test_facility(21))
-                        /* llilf %r5,<K> */
-                        EMIT6_IMM(0xc05f0000, K);
-                else
-                        /* l %r5,<d(K)>(%r13) */
-                        EMIT4_DISP(0x5850d000, EMIT_CONST(K));
-                break;
-        case BPF_LDX | BPF_IMM: /* X = K */
-                jit->seen |= SEEN_XREG;
-                if (K <= 16383)
-                        /* lhi %r12,<K> */
-                        EMIT4_IMM(0xa7c80000, K);
-                else if (test_facility(21))
-                        /* llilf %r12,<K> */
-                        EMIT6_IMM(0xc0cf0000, K);
-                else
-                        /* l %r12,<d(K)>(%r13) */
-                        EMIT4_DISP(0x58c0d000, EMIT_CONST(K));
                break;
-        case BPF_LD | BPF_MEM: /* A = mem[K] */
+        /*
+         * BPF_ST(X)
+         */
+        case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */
+                /* stcy %src,off(%dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
+                /* sthy %src,off(%dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
+                /* sty %src,off(%dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
+                /* stg %src,off(%dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
+                /* lhi %w0,imm */
+                EMIT4_IMM(0xa7080000, REG_W0, (u8) imm);
+                /* stcy %w0,off(dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
+                /* lhi %w0,imm */
+                EMIT4_IMM(0xa7080000, REG_W0, (u16) imm);
+                /* sthy %w0,off(dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, REG_0, off);
                jit->seen |= SEEN_MEM;
-                /* l %r5,<K>(%r15) */
-                EMIT4_DISP(0x5850f000,
-                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
-                break;
-        case BPF_LDX | BPF_MEM: /* X = mem[K] */
-                jit->seen |= SEEN_XREG | SEEN_MEM;
-                /* l %r12,<K>(%r15) */
-                EMIT4_DISP(0x58c0f000,
-                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
-                break;
-        case BPF_MISC | BPF_TAX: /* X = A */
-                jit->seen |= SEEN_XREG;
-                /* lr %r12,%r5 */
-                EMIT2(0x18c5);
-                break;
-        case BPF_MISC | BPF_TXA: /* A = X */
-                jit->seen |= SEEN_XREG;
-                /* lr %r5,%r12 */
-                EMIT2(0x185c);
-                break;
-        case BPF_RET | BPF_K:
-                if (K == 0) {
-                        jit->seen |= SEEN_RET0;
-                        if (last)
-                                break;
-                        /* j <ret0> */
-                        EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg);
-                } else {
-                        if (K <= 16383)
-                                /* lghi %r2,K */
-                                EMIT4_IMM(0xa7290000, K);
-                        else
-                                /* llgf %r2,<K>(%r13) */
-                                EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K));
-                        /* j <exit> */
-                        if (last && !(jit->seen & SEEN_RET0))
-                                break;
-                        EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);
-                }
                break;
-        case BPF_RET | BPF_A:
+        case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
-                /* llgfr %r2,%r5 */
+                /* llilf %w0,imm  */
-                EMIT4(0xb9160025);
+                EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm);
+                /* sty %w0,off(%dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
+                /* lgfi %w0,imm */
+                EMIT6_IMM(0xc0010000, REG_W0, imm);
+                /* stg %w0,off(%dst) */
+                EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        /*
+         * BPF_STX XADD (atomic_add)
+         */
+        case BPF_STX | BPF_XADD | BPF_W: /* *(u32 *)(dst + off) += src */
+                /* laal %w0,%src,off(%dst) */
+                EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W0, src_reg,
+                              dst_reg, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_STX | BPF_XADD | BPF_DW: /* *(u64 *)(dst + off) += src */
+                /* laalg %w0,%src,off(%dst) */
+                EMIT6_DISP_LH(0xeb000000, 0x00ea, REG_W0, src_reg,
+                              dst_reg, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        /*
+         * BPF_LDX
+         */
+        case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
+                /* llgc %dst,0(off,%src) */
+                EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
+                /* llgh %dst,0(off,%src) */
+                EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);
+                jit->seen |= SEEN_MEM;
+                break;
+        case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
+                /* llgf %dst,off(%src) */
+                jit->seen |= SEEN_MEM;
+                EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
+                break;
+        case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
+                /* lg %dst,0(off,%src) */
+                jit->seen |= SEEN_MEM;
+                EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
+                break;
+        /*
+         * BPF_JMP / CALL
+         */
+        case BPF_JMP | BPF_CALL:
+        {
+                /*
+                 * b0 = (__bpf_call_base + imm)(b1, b2, b3, b4, b5)
+                 */
+                const u64 func = (u64)__bpf_call_base + imm;
+                REG_SET_SEEN(BPF_REG_5);
+                jit->seen |= SEEN_FUNC;
+                /* lg %w1,<d(imm)>(%l) */
+                EMIT6_DISP(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
+                           EMIT_CONST_U64(func));
+                /* basr %r14,%w1 */
+                EMIT2(0x0d00, REG_14, REG_W1);
+                /* lgr %b0,%r2: load return value into %b0 */
+                EMIT4(0xb9040000, BPF_REG_0, REG_2);
+                break;
+        }
+        case BPF_JMP | BPF_EXIT: /* return b0 */
+                last = (i == fp->len - 1) ? 1 : 0;
+                if (last && !(jit->seen & SEEN_RET0))
+                        break;
                /* j <exit> */
                EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);
                break;
-        case BPF_ST: /* mem[K] = A */
+        /*
-                jit->seen |= SEEN_MEM;
+         * Branch relative (number of skipped instructions) to offset on
-                /* st %r5,<K>(%r15) */
+         * condition.
-                EMIT4_DISP(0x5050f000,
+         *
-                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
+         * Condition code to mask mapping:
-                break;
+         *
-        case BPF_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */
+         * CC | Description        | Mask
-                jit->seen |= SEEN_XREG | SEEN_MEM;
+         * ------------------------------
-                /* st %r12,<K>(%r15) */
+         * 0  | Operands equal     |    8
-                EMIT4_DISP(0x50c0f000,
+         * 1  | First operand low  |    4
-                           (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
+         * 2  | First operand high |    2
-                break;
+         * 3  | Unused             |    1
-        case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */
+         *
-                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
+         * For s390x relative branches: ip = ip + off_bytes
-                /* lhi %r5,0 */
+         * For BPF relative branches:   insn = insn + off_insns + 1
-                EMIT4(0xa7580000);
+         *
-                /* icm  %r5,3,<d(protocol)>(%r2) */
+         * For example for s390x with offset 0 we jump to the branch
-                EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol));
+         * instruction itself (loop) and for BPF with offset 0 we
-                break;
+         * branch to the instruction behind the branch.
-        case BPF_ANC | SKF_AD_IFINDEX:  /* if (!skb->dev) return 0;
+         */
-                                         * A = skb->dev->ifindex */
+        case BPF_JMP | BPF_JA: /* if (true) */
-                BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
+                mask = 0xf000; /* j */
-                jit->seen |= SEEN_RET0;
+                goto branch_oc;
-                /* lg %r1,<d(dev)>(%r2) */
+        case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */
-                EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev));
+                mask = 0x2000; /* jh */
-                /* ltgr %r1,%r1 */
+                goto branch_ks;
-                EMIT4(0xb9020011);
+        case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */
-                /* jz <ret0> */
+                mask = 0xa000; /* jhe */
-                EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
+                goto branch_ks;
-                /* l %r5,<d(ifindex)>(%r1) */
+        case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */
-                EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex));
+                mask = 0x2000; /* jh */
-                break;
+                goto branch_ku;
-        case BPF_ANC | SKF_AD_MARK: /* A = skb->mark */
+        case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */
-                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
+                mask = 0xa000; /* jhe */
-                /* l %r5,<d(mark)>(%r2) */
+                goto branch_ku;
-                EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark));
+        case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */
-                break;
+                mask = 0x7000; /* jne */
-        case BPF_ANC | SKF_AD_QUEUE: /* A = skb->queue_mapping */
+                goto branch_ku;
-                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
+        case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */
-                /* lhi %r5,0 */
+                mask = 0x8000; /* je */
-                EMIT4(0xa7580000);
+                goto branch_ku;
-                /* icm  %r5,3,<d(queue_mapping)>(%r2) */
+        case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */
-                EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping));
+                mask = 0x7000; /* jnz */
-                break;
+                /* lgfi %w1,imm (load sign extend imm) */
-        case BPF_ANC | SKF_AD_HATYPE:   /* if (!skb->dev) return 0;
+                EMIT6_IMM(0xc0010000, REG_W1, imm);
-                                         * A = skb->dev->type */
+                /* ngr %w1,%dst */
-                BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
+                EMIT4(0xb9800000, REG_W1, dst_reg);
-                jit->seen |= SEEN_RET0;
+                goto branch_oc;
-                /* lg %r1,<d(dev)>(%r2) */
-                EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev));
+        case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */
-                /* ltgr %r1,%r1 */
+                mask = 0x2000; /* jh */
-                EMIT4(0xb9020011);
+                goto branch_xs;
-                /* jz <ret0> */
+        case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */
-                EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
+                mask = 0xa000; /* jhe */
-                /* lhi %r5,0 */
+                goto branch_xs;
-                EMIT4(0xa7580000);
+        case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */
-                /* icm  %r5,3,<d(type)>(%r1) */
+                mask = 0x2000; /* jh */
-                EMIT4_DISP(0xbf531000, offsetof(struct net_device, type));
+                goto branch_xu;
-                break;
+        case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */
-        case BPF_ANC | SKF_AD_RXHASH: /* A = skb->hash */
+                mask = 0xa000; /* jhe */
-                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
+                goto branch_xu;
-                /* l %r5,<d(hash)>(%r2) */
+        case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */
-                EMIT4_DISP(0x58502000, offsetof(struct sk_buff, hash));
+                mask = 0x7000; /* jne */
-                break;
+                goto branch_xu;
-        case BPF_ANC | SKF_AD_VLAN_TAG:
+        case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */
-        case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT:
+                mask = 0x8000; /* je */
-                BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
+                goto branch_xu;
-                BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
+        case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */
-                /* lhi %r5,0 */
+                mask = 0x7000; /* jnz */
-                EMIT4(0xa7580000);
+                /* ngrk %w1,%dst,%src */
-                /* icm  %r5,3,<d(vlan_tci)>(%r2) */
+                EMIT4_RRF(0xb9e40000, REG_W1, dst_reg, src_reg);
-                EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci));
+                goto branch_oc;
-                if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) {
+branch_ks:
-                        /* nill %r5,0xefff */
+                /* lgfi %w1,imm (load sign extend imm) */
-                        EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT);
+                EMIT6_IMM(0xc0010000, REG_W1, imm);
-                } else {
+                /* cgrj %dst,%w1,mask,off */
-                        /* nill %r5,0x1000 */
+                EMIT6_PCREL(0xec000000, 0x0064, dst_reg, REG_W1, i, off, mask);
-                        EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT);
+                break;
-                        /* srl %r5,12 */
+branch_ku:
-                        EMIT4_DISP(0x88500000, 12);
+                /* lgfi %w1,imm (load sign extend imm) */
-                }
+                EMIT6_IMM(0xc0010000, REG_W1, imm);
+                /* clgrj %dst,%w1,mask,off */
+                EMIT6_PCREL(0xec000000, 0x0065, dst_reg, REG_W1, i, off, mask);
+                break;
+branch_xs:
+                /* cgrj %dst,%src,mask,off */
+                EMIT6_PCREL(0xec000000, 0x0064, dst_reg, src_reg, i, off, mask);
+                break;
+branch_xu:
+                /* clgrj %dst,%src,mask,off */
+                EMIT6_PCREL(0xec000000, 0x0065, dst_reg, src_reg, i, off, mask);
+                break;
+branch_oc:
+                /* brc mask,jmp_off (branch instruction needs 4 bytes) */
+                jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
+                EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);
                break;
-        case BPF_ANC | SKF_AD_PKTTYPE:
+        /*
-                /* lhi %r5,0 */
+         * BPF_LD
-                EMIT4(0xa7580000);
+         */
-                /* ic %r5,<d(pkt_type_offset)>(%r2) */
+        case BPF_LD | BPF_ABS | BPF_B: /* b0 = *(u8 *) (skb->data+imm) */
-                EMIT4_DISP(0x43502000, PKT_TYPE_OFFSET());
+        case BPF_LD | BPF_IND | BPF_B: /* b0 = *(u8 *) (skb->data+imm+src) */
-                /* srl %r5,5 */
+                if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
-                EMIT4_DISP(0x88500000, 5);
+                        func_addr = __pa(sk_load_byte_pos);
-                break;
+                else
-        case BPF_ANC | SKF_AD_CPU: /* A = smp_processor_id() */
+                        func_addr = __pa(sk_load_byte);
-#ifdef CONFIG_SMP
+                goto call_fn;
-                /* l %r5,<d(cpu_nr)> */
+        case BPF_LD | BPF_ABS | BPF_H: /* b0 = *(u16 *) (skb->data+imm) */
-                EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr));
+        case BPF_LD | BPF_IND | BPF_H: /* b0 = *(u16 *) (skb->data+imm+src) */
-#else
+                if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
-                /* lhi %r5,0 */
+                        func_addr = __pa(sk_load_half_pos);
-                EMIT4(0xa7580000);
+                else
-#endif
+                        func_addr = __pa(sk_load_half);
+                goto call_fn;
+        case BPF_LD | BPF_ABS | BPF_W: /* b0 = *(u32 *) (skb->data+imm) */
+        case BPF_LD | BPF_IND | BPF_W: /* b0 = *(u32 *) (skb->data+imm+src) */
+                if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
+                        func_addr = __pa(sk_load_word_pos);
+                else
+                        func_addr = __pa(sk_load_word);
+                goto call_fn;
+call_fn:
+                jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC;
+                REG_SET_SEEN(REG_14); /* Return address of possible func call */
+                /*
+                 * Implicit input:
+                 *  BPF_REG_6    (R7) : skb pointer
+                 *  REG_SKB_DATA (R12): skb data pointer
+                 *
+                 * Calculated input:
+                 *  BPF_REG_2    (R3) : offset of byte(s) to fetch in skb
+                 *  BPF_REG_5    (R6) : return address
+                 *
+                 * Output:
+                 *  BPF_REG_0    (R14): data read from skb
+                 *
+                 * Scratch registers (BPF_REG_1-5)
+                 */
+                /* Call function: llilf %w1,func_addr  */
+                EMIT6_IMM(0xc00f0000, REG_W1, func_addr);
+                /* Offset: lgfi %b2,imm */
+                EMIT6_IMM(0xc0010000, BPF_REG_2, imm);
+                if (BPF_MODE(insn->code) == BPF_IND)
+                        /* agfr %b2,%src (%src is s32 here) */
+                        EMIT4(0xb9180000, BPF_REG_2, src_reg);
+                /* basr %b5,%w1 (%b5 is call saved) */
+                EMIT2(0x0d00, BPF_REG_5, REG_W1);
+                /*
+                 * Note: For fast access we jump directly after the
+                 * jnz instruction from bpf_jit.S
+                 */
+                /* jnz <ret0> */
+                EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);
                break;
        default: /* too complex, give up */
-                goto out;
+                pr_err("Unknown opcode %02x\n", insn->code);
+                return -1;
+        }
+        return insn_count;
+}
+/*
+ * Compile eBPF program into s390x code
+ */
+static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp)
+{
+        int i, insn_count;
+        jit->lit = jit->lit_start;
+        jit->prg = 0;
+        bpf_jit_prologue(jit);
+        for (i = 0; i < fp->len; i += insn_count) {
+                insn_count = bpf_jit_insn(jit, fp, i);
+                if (insn_count < 0)
+                        return -1;
+                jit->addrs[i + 1] = jit->prg; /* Next instruction address */
        }
-        addrs[i] = jit->prg - jit->start;
+        bpf_jit_epilogue(jit);
+        jit->lit_start = jit->prg;
+        jit->size = jit->lit;
+        jit->size_prg = jit->prg;
        return 0;
-out:
-        return -1;
 }
+/*
+ * Classic BPF function stub. BPF programs will be converted into
+ * eBPF and then bpf_int_jit_compile() will be called.
+ */
 void bpf_jit_compile(struct bpf_prog *fp)
 {
-        struct bpf_binary_header *header = NULL;
+}
-        unsigned long size, prg_len, lit_len;
-        struct bpf_jit jit, cjit;
+/*
-        unsigned int *addrs;
+ * Compile eBPF program "fp"
-        int pass, i;
+ */
+void bpf_int_jit_compile(struct bpf_prog *fp)
+{
+        struct bpf_binary_header *header;
+        struct bpf_jit jit;
+        int pass;
        if (!bpf_jit_enable)
                return;
-        addrs = kcalloc(fp->len, sizeof(*addrs), GFP_KERNEL);
+        memset(&jit, 0, sizeof(jit));
-        if (addrs == NULL)
+        jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
+        if (jit.addrs == NULL)
                return;
-        memset(&jit, 0, sizeof(cjit));
+        /*
-        memset(&cjit, 0, sizeof(cjit));
+         * Three initial passes:
+         *   - 1/2: Determine clobbered registers
-        for (pass = 0; pass < 10; pass++) {
+         *   - 3:   Calculate program size and addrs arrray
-                jit.prg = jit.start;
+         */
-                jit.lit = jit.mid;
+        for (pass = 1; pass <= 3; pass++) {
+                if (bpf_jit_prog(&jit, fp))
-                bpf_jit_prologue(&jit);
+                        goto free_addrs;
-                bpf_jit_noleaks(&jit, fp->insns);
-                for (i = 0; i < fp->len; i++) {
-                        if (bpf_jit_insn(&jit, fp->insns + i, addrs, i,
-                                         i == fp->len - 1))
-                                goto out;
-                }
-                bpf_jit_epilogue(&jit);
-                if (jit.start) {
-                        WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit);
-                        if (memcmp(&jit, &cjit, sizeof(jit)) == 0)
-                                break;
-                } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) {
-                        prg_len = jit.prg - jit.start;
-                        lit_len = jit.lit - jit.mid;
-                        size = prg_len + lit_len;
-                        if (size >= BPF_SIZE_MAX)
-                                goto out;
-                        header = bpf_jit_binary_alloc(size, &jit.start,
-                                                      2, bpf_jit_fill_hole);
-                        if (!header)
-                                goto out;
-                        jit.prg = jit.mid = jit.start + prg_len;
-                        jit.lit = jit.end = jit.start + prg_len + lit_len;
-                        jit.base_ip += (unsigned long) jit.start;
-                        jit.exit_ip += (unsigned long) jit.start;
-                        jit.ret0_ip += (unsigned long) jit.start;
-                }
-                cjit = jit;
        }
+        /*
+         * Final pass: Allocate and generate program
+         */
+        if (jit.size >= BPF_SIZE_MAX)
+                goto free_addrs;
+        header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);
+        if (!header)
+                goto free_addrs;
+        if (bpf_jit_prog(&jit, fp))
+                goto free_addrs;
        if (bpf_jit_enable > 1) {
-                bpf_jit_dump(fp->len, jit.end - jit.start, pass, jit.start);
+                bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
-                if (jit.start)
+                if (jit.prg_buf)
-                        print_fn_code(jit.start, jit.mid - jit.start);
+                        print_fn_code(jit.prg_buf, jit.size_prg);
        }
-        if (jit.start) {
+        if (jit.prg_buf) {
                set_memory_ro((unsigned long)header, header->pages);
-                fp->bpf_func = (void *) jit.start;
+                fp->bpf_func = (void *) jit.prg_buf;
                fp->jited = true;
        }
-out:
+free_addrs:
-        kfree(addrs);
+        kfree(jit.addrs);
 }
+/*
+ * Free eBPF program
+ */
 void bpf_jit_free(struct bpf_prog *fp)
 {
        unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
index 98336200c7b2..598f023cf8a6 100644
--- a/arch/s390/pci/pci.c
+++ b/arch/s390/pci/pci.c
@@ -190,6 +190,11 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev)
                return -ENOMEM;
        WARN_ON((u64) zdev->fmb & 0xf);
+        /* reset software counters */
+        atomic64_set(&zdev->allocated_pages, 0);
+        atomic64_set(&zdev->mapped_pages, 0);
+        atomic64_set(&zdev->unmapped_pages, 0);
        args.fmb_addr = virt_to_phys(zdev->fmb);
        return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args);
 }
@@ -822,6 +827,7 @@ int zpci_create_device(struct zpci_dev *zdev)
        if (rc)
                goto out;
+        mutex_init(&zdev->lock);
        if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
                rc = zpci_enable_device(zdev);
                if (rc)
diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c
index c22d4402ae45..4129b0a5fd78 100644
--- a/arch/s390/pci/pci_debug.c
+++ b/arch/s390/pci/pci_debug.c
@@ -31,12 +31,25 @@ static char *pci_perf_names[] = {
        "Refresh operations",
        "DMA read bytes",
        "DMA write bytes",
-        /* software counters */
+};
+static char *pci_sw_names[] = {
        "Allocated pages",
        "Mapped pages",
        "Unmapped pages",
 };
+static void pci_sw_counter_show(struct seq_file *m)
+{
+        struct zpci_dev *zdev = m->private;
+        atomic64_t *counter = &zdev->allocated_pages;
+        int i;
+        for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++)
+                seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i],
+                           atomic64_read(counter));
+}
 static int pci_perf_show(struct seq_file *m, void *v)
 {
        struct zpci_dev *zdev = m->private;
@@ -45,7 +58,10 @@ static int pci_perf_show(struct seq_file *m, void *v)
        if (!zdev)
                return 0;
+        mutex_lock(&zdev->lock);
        if (!zdev->fmb) {
+                mutex_unlock(&zdev->lock);
                seq_puts(m, "FMB statistics disabled\n");
                return 0;
        }
@@ -65,12 +81,9 @@ static int pci_perf_show(struct seq_file *m, void *v)
                for (i = 4; i < 6; i++)
                        seq_printf(m, "%26s:\t%llu\n",
                                   pci_perf_names[i], *(stat + i));
-        /* software counters */
-        for (i = 6; i < ARRAY_SIZE(pci_perf_names); i++)
-                seq_printf(m, "%26s:\t%llu\n",
-                           pci_perf_names[i],
-                           atomic64_read((atomic64_t *) (stat + i)));
+        pci_sw_counter_show(m);
+        mutex_unlock(&zdev->lock);
        return 0;
 }
@@ -88,19 +101,17 @@ static ssize_t pci_perf_seq_write(struct file *file, const char __user *ubuf,
        if (rc)
                return rc;
+        mutex_lock(&zdev->lock);
        switch (val) {
        case 0:
                rc = zpci_fmb_disable_device(zdev);
-                if (rc)
-                        return rc;
                break;
        case 1:
                rc = zpci_fmb_enable_device(zdev);
-                if (rc)
-                        return rc;
                break;
        }
-        return count;
+        mutex_unlock(&zdev->lock);
+        return rc ? rc : count;
 }
 static int pci_perf_seq_open(struct inode *inode, struct file *filp)
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index 4cbb29a4d615..6fd8d5836138 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -300,7 +300,7 @@ static dma_addr_t s390_dma_map_pages(struct device *dev, struct page *page,
                flags |= ZPCI_TABLE_PROTECTED;
        if (!dma_update_trans(zdev, pa, dma_addr, size, flags)) {
-                atomic64_add(nr_pages, &zdev->fmb->mapped_pages);
+                atomic64_add(nr_pages, &zdev->mapped_pages);
                return dma_addr + (offset & ~PAGE_MASK);
        }
@@ -328,7 +328,7 @@ static void s390_dma_unmap_pages(struct device *dev, dma_addr_t dma_addr,
                zpci_err_hex(&dma_addr, sizeof(dma_addr));
        }
-        atomic64_add(npages, &zdev->fmb->unmapped_pages);
+        atomic64_add(npages, &zdev->unmapped_pages);
        iommu_page_index = (dma_addr - zdev->start_dma) >> PAGE_SHIFT;
        dma_free_iommu(zdev, iommu_page_index, npages);
 }
@@ -357,7 +357,7 @@ static void *s390_dma_alloc(struct device *dev, size_t size,
                return NULL;
        }
-        atomic64_add(size / PAGE_SIZE, &zdev->fmb->allocated_pages);
+        atomic64_add(size / PAGE_SIZE, &zdev->allocated_pages);
        if (dma_handle)
                *dma_handle = map;
        return (void *) pa;
@@ -370,7 +370,7 @@ static void s390_dma_free(struct device *dev, size_t size,
        struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));
        size = PAGE_ALIGN(size);
-        atomic64_sub(size / PAGE_SIZE, &zdev->fmb->allocated_pages);
+        atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages);
        s390_dma_unmap_pages(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
        free_pages((unsigned long) pa, get_order(size));
 }
diff --git a/arch/sh/include/asm/mmu_context.h b/arch/sh/include/asm/mmu_context.h
index b9d9489a5012..9f417feaf6e8 100644
--- a/arch/sh/include/asm/mmu_context.h
+++ b/arch/sh/include/asm/mmu_context.h
@@ -99,7 +99,7 @@ static inline int init_new_context(struct task_struct *tsk,
 {
        int i;
-        for (i = 0; i < num_online_cpus(); i++)
+        for_each_online_cpu(i)
                cpu_context(i, mm) = NO_CONTEXT;
        return 0;
diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c
index fc5acfc93c92..de6be008fc01 100644
--- a/arch/sh/kernel/smp.c
+++ b/arch/sh/kernel/smp.c
@@ -363,7 +363,7 @@ void flush_tlb_mm(struct mm_struct *mm)
                smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1);
        } else {
                int i;
-                for (i = 0; i < num_online_cpus(); i++)
+                for_each_online_cpu(i)
                        if (smp_processor_id() != i)
                                cpu_context(i, mm) = 0;
        }
@@ -400,7 +400,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
                smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1);
        } else {
                int i;
-                for (i = 0; i < num_online_cpus(); i++)
+                for_each_online_cpu(i)
                        if (smp_processor_id() != i)
                                cpu_context(i, mm) = 0;
        }
@@ -443,7 +443,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
                smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1);
        } else {
                int i;
-                for (i = 0; i < num_online_cpus(); i++)
+                for_each_online_cpu(i)
                        if (smp_processor_id() != i)
                                cpu_context(i, vma->vm_mm) = 0;
        }
diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c
index 18147a5523d9..8caf45ee81d9 100644
--- a/arch/sparc/kernel/time_32.c
+++ b/arch/sparc/kernel/time_32.c
@@ -194,7 +194,7 @@ static __init int setup_timer_cs(void)
 static void percpu_ce_setup(enum clock_event_mode mode,
                        struct clock_event_device *evt)
 {
-        int cpu = __first_cpu(evt->cpumask);
+        int cpu = cpumask_first(evt->cpumask);
        switch (mode) {
                case CLOCK_EVT_MODE_PERIODIC:
@@ -214,7 +214,7 @@ static void percpu_ce_setup(enum clock_event_mode mode,
 static int percpu_ce_set_next_event(unsigned long delta,
                                    struct clock_event_device *evt)
 {
-        int cpu = __first_cpu(evt->cpumask);
+        int cpu = cpumask_first(evt->cpumask);
        unsigned int next = (unsigned int)delta;
        sparc_config.load_profile_irq(cpu, next);
diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c
index 7833b2ccdfbc..6873f006f7d0 100644
--- a/arch/tile/kernel/setup.c
+++ b/arch/tile/kernel/setup.c
@@ -774,7 +774,7 @@ static void __init zone_sizes_init(void)
                 * though, there'll be no lowmem, so we just alloc_bootmem
                 * the memmap.  There will be no percpu memory either.
                 */
-                if (i != 0 && cpu_isset(i, isolnodes)) {
+                if (i != 0 && cpumask_test_cpu(i, &isolnodes)) {
                        node_memmap_pfn[i] =
                                alloc_bootmem_pfn(0, memmap_size, 0);
                        BUG_ON(node_percpu[i] != 0);
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index d9d0bd2faaf4..ab3219b3fbda 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -171,8 +171,8 @@ update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu)
                for_each_online_cpu(cpu) {
                        if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
                                continue;
-                        __cpu_clear(this_cpu, per_cpu(cpus_in_cluster, cpu));
+                        cpumask_clear_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu));
-                        __cpu_clear(cpu, per_cpu(cpus_in_cluster, this_cpu));
+                        cpumask_clear_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu));
                }
                free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
                free_cpumask_var(per_cpu(ipi_mask, this_cpu));
diff --git a/drivers/clocksource/dw_apb_timer.c b/drivers/clocksource/dw_apb_timer.c
index f3656a6b0382..35a88097af3c 100644
--- a/drivers/clocksource/dw_apb_timer.c
+++ b/drivers/clocksource/dw_apb_timer.c
@@ -117,7 +117,8 @@ static void apbt_set_mode(enum clock_event_mode mode,
        unsigned long period;
        struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt);
-        pr_debug("%s CPU %d mode=%d\n", __func__, first_cpu(*evt->cpumask),
+        pr_debug("%s CPU %d mode=%d\n", __func__,
+                 cpumask_first(evt->cpumask),
                 mode);
        switch (mode) {
diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c
index 73fe2f8d7f96..7936dce4b878 100644
--- a/drivers/cpuidle/coupled.c
+++ b/drivers/cpuidle/coupled.c
@@ -292,7 +292,7 @@ static inline int cpuidle_coupled_get_state(struct cpuidle_device *dev,
         */
        smp_rmb();
-        for_each_cpu_mask(i, coupled->coupled_cpus)
+        for_each_cpu(i, &coupled->coupled_cpus)
                if (cpu_online(i) && coupled->requested_state[i] < state)
                        state = coupled->requested_state[i];
@@ -338,7 +338,7 @@ static void cpuidle_coupled_poke_others(int this_cpu,
 {
        int cpu;
-        for_each_cpu_mask(cpu, coupled->coupled_cpus)
+        for_each_cpu(cpu, &coupled->coupled_cpus)
                if (cpu != this_cpu && cpu_online(cpu))
                        cpuidle_coupled_poke(cpu);
 }
@@ -638,7 +638,7 @@ int cpuidle_coupled_register_device(struct cpuidle_device *dev)
        if (cpumask_empty(&dev->coupled_cpus))
                return 0;
-        for_each_cpu_mask(cpu, dev->coupled_cpus) {
+        for_each_cpu(cpu, &dev->coupled_cpus) {
                other_dev = per_cpu(cpuidle_devices, cpu);
                if (other_dev && other_dev->coupled) {
                        coupled = other_dev->coupled;
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
index afd136b45f49..10a9aeff1666 100644
--- a/drivers/crypto/n2_core.c
+++ b/drivers/crypto/n2_core.c
@@ -1754,7 +1754,7 @@ static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
                                dev->dev.of_node->full_name);
                        return -EINVAL;
                }
-                cpu_set(*id, p->sharing);
+                cpumask_set_cpu(*id, &p->sharing);
                table[*id] = p;
        }
        return 0;
@@ -1776,7 +1776,7 @@ static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
                return -ENOMEM;
        }
-        cpus_clear(p->sharing);
+        cpumask_clear(&p->sharing);
        spin_lock_init(&p->lock);
        p->q_type = q_type;
        INIT_LIST_HEAD(&p->jobs);
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 48882c126245..e43d48956dea 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -33,6 +33,7 @@
 #include <linux/export.h>
 #include <linux/irq.h>
 #include <linux/msi.h>
+#include <linux/dma-contiguous.h>
 #include <asm/irq_remapping.h>
 #include <asm/io_apic.h>
 #include <asm/apic.h>
@@ -126,6 +127,11 @@ static int __init alloc_passthrough_domain(void);
 *
 ****************************************************************************/
+static struct protection_domain *to_pdomain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct protection_domain, domain);
+}
 static struct iommu_dev_data *alloc_dev_data(u16 devid)
 {
        struct iommu_dev_data *dev_data;
@@ -1321,7 +1327,9 @@ static u64 *alloc_pte(struct protection_domain *domain,
 * This function checks if there is a PTE for a given dma address. If
 * there is one, it returns the pointer to it.
 */
-static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
+static u64 *fetch_pte(struct protection_domain *domain,
+                      unsigned long address,
+                      unsigned long *page_size)
 {
        int level;
        u64 *pte;
@@ -1329,8 +1337,9 @@ static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
        if (address > PM_LEVEL_SIZE(domain->mode))
                return NULL;
-        level   =  domain->mode - 1;
+        level      =  domain->mode - 1;
-        pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
+        pte        = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
+        *page_size =  PTE_LEVEL_PAGE_SIZE(level);
        while (level > 0) {
@@ -1339,19 +1348,9 @@ static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
                        return NULL;
                /* Large PTE */
-                if (PM_PTE_LEVEL(*pte) == 0x07) {
+                if (PM_PTE_LEVEL(*pte) == 7 ||
-                        unsigned long pte_mask, __pte;
+                    PM_PTE_LEVEL(*pte) == 0)
+                        break;
-                        /*
-                         * If we have a series of large PTEs, make
-                         * sure to return a pointer to the first one.
-                         */
-                        pte_mask = PTE_PAGE_SIZE(*pte);
-                        pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
-                        __pte    = ((unsigned long)pte) & pte_mask;
-                        return (u64 *)__pte;
-                }
                /* No level skipping support yet */
                if (PM_PTE_LEVEL(*pte) != level)
@@ -1360,8 +1359,21 @@ static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
                level -= 1;
                /* Walk to the next level */
-                pte = IOMMU_PTE_PAGE(*pte);
+                pte        = IOMMU_PTE_PAGE(*pte);
-                pte = &pte[PM_LEVEL_INDEX(level, address)];
+                pte        = &pte[PM_LEVEL_INDEX(level, address)];
+                *page_size = PTE_LEVEL_PAGE_SIZE(level);
+        }
+        if (PM_PTE_LEVEL(*pte) == 0x07) {
+                unsigned long pte_mask;
+                /*
+                 * If we have a series of large PTEs, make
+                 * sure to return a pointer to the first one.
+                 */
+                *page_size = pte_mask = PTE_PAGE_SIZE(*pte);
+                pte_mask   = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
+                pte        = (u64 *)(((unsigned long)pte) & pte_mask);
        }
        return pte;
@@ -1383,13 +1395,14 @@ static int iommu_map_page(struct protection_domain *dom,
        u64 __pte, *pte;
        int i, count;
+        BUG_ON(!IS_ALIGNED(bus_addr, page_size));
+        BUG_ON(!IS_ALIGNED(phys_addr, page_size));
        if (!(prot & IOMMU_PROT_MASK))
                return -EINVAL;
-        bus_addr  = PAGE_ALIGN(bus_addr);
+        count = PAGE_SIZE_PTE_COUNT(page_size);
-        phys_addr = PAGE_ALIGN(phys_addr);
+        pte   = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
-        count     = PAGE_SIZE_PTE_COUNT(page_size);
-        pte       = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
        if (!pte)
                return -ENOMEM;
@@ -1398,7 +1411,7 @@ static int iommu_map_page(struct protection_domain *dom,
                if (IOMMU_PTE_PRESENT(pte[i]))
                        return -EBUSY;
-        if (page_size > PAGE_SIZE) {
+        if (count > 1) {
                __pte = PAGE_SIZE_PTE(phys_addr, page_size);
                __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
        } else
@@ -1421,7 +1434,8 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,
                                      unsigned long bus_addr,
                                      unsigned long page_size)
 {
-        unsigned long long unmap_size, unmapped;
+        unsigned long long unmapped;
+        unsigned long unmap_size;
        u64 *pte;
        BUG_ON(!is_power_of_2(page_size));
@@ -1430,28 +1444,12 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,
        while (unmapped < page_size) {
-                pte = fetch_pte(dom, bus_addr);
+                pte = fetch_pte(dom, bus_addr, &unmap_size);
-                if (!pte) {
+                if (pte) {
-                        /*
+                        int i, count;
-                         * No PTE for this address
-                         * move forward in 4kb steps
+                        count = PAGE_SIZE_PTE_COUNT(unmap_size);
-                         */
-                        unmap_size = PAGE_SIZE;
-                } else if (PM_PTE_LEVEL(*pte) == 0) {
-                        /* 4kb PTE found for this address */
-                        unmap_size = PAGE_SIZE;
-                        *pte       = 0ULL;
-                } else {
-                        int count, i;
-                        /* Large PTE found which maps this address */
-                        unmap_size = PTE_PAGE_SIZE(*pte);
-                        /* Only unmap from the first pte in the page */
-                        if ((unmap_size - 1) & bus_addr)
-                                break;
-                        count      = PAGE_SIZE_PTE_COUNT(unmap_size);
                        for (i = 0; i < count; i++)
                                pte[i] = 0ULL;
                }
@@ -1599,7 +1597,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
 {
        int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
        struct amd_iommu *iommu;
-        unsigned long i, old_size;
+        unsigned long i, old_size, pte_pgsize;
 #ifdef CONFIG_IOMMU_STRESS
        populate = false;
@@ -1672,12 +1670,13 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
         */
        for (i = dma_dom->aperture[index]->offset;
             i < dma_dom->aperture_size;
-             i += PAGE_SIZE) {
+             i += pte_pgsize) {
-                u64 *pte = fetch_pte(&dma_dom->domain, i);
+                u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);
                if (!pte || !IOMMU_PTE_PRESENT(*pte))
                        continue;
-                dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
+                dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT,
+                                          pte_pgsize >> 12);
        }
        update_domain(&dma_dom->domain);
@@ -2422,16 +2421,6 @@ static int device_change_notifier(struct notifier_block *nb,
        dev_data = get_dev_data(dev);
        switch (action) {
-        case BUS_NOTIFY_UNBOUND_DRIVER:
-                domain = domain_for_device(dev);
-                if (!domain)
-                        goto out;
-                if (dev_data->passthrough)
-                        break;
-                detach_device(dev);
-                break;
        case BUS_NOTIFY_ADD_DEVICE:
                iommu_init_device(dev);
@@ -2467,7 +2456,7 @@ static int device_change_notifier(struct notifier_block *nb,
                dev->archdata.dma_ops = &amd_iommu_dma_ops;
                break;
-        case BUS_NOTIFY_DEL_DEVICE:
+        case BUS_NOTIFY_REMOVED_DEVICE:
                iommu_uninit_device(dev);
@@ -2923,38 +2912,42 @@ static void *alloc_coherent(struct device *dev, size_t size,
                            dma_addr_t *dma_addr, gfp_t flag,
                            struct dma_attrs *attrs)
 {
-        unsigned long flags;
-        void *virt_addr;
-        struct protection_domain *domain;
-        phys_addr_t paddr;
        u64 dma_mask = dev->coherent_dma_mask;
+        struct protection_domain *domain;
+        unsigned long flags;
+        struct page *page;
        INC_STATS_COUNTER(cnt_alloc_coherent);
        domain = get_domain(dev);
        if (PTR_ERR(domain) == -EINVAL) {
-                virt_addr = (void *)__get_free_pages(flag, get_order(size));
+                page = alloc_pages(flag, get_order(size));
-                *dma_addr = __pa(virt_addr);
+                *dma_addr = page_to_phys(page);
-                return virt_addr;
+                return page_address(page);
        } else if (IS_ERR(domain))
                return NULL;
+        size      = PAGE_ALIGN(size);
        dma_mask  = dev->coherent_dma_mask;
        flag     &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
-        flag     |= __GFP_ZERO;
-        virt_addr = (void *)__get_free_pages(flag, get_order(size));
+        page = alloc_pages(flag | __GFP_NOWARN,  get_order(size));
-        if (!virt_addr)
+        if (!page) {
-                return NULL;
+                if (!(flag & __GFP_WAIT))
+                        return NULL;
-        paddr = virt_to_phys(virt_addr);
+                page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
+                                                 get_order(size));
+                if (!page)
+                        return NULL;
+        }
        if (!dma_mask)
                dma_mask = *dev->dma_mask;
        spin_lock_irqsave(&domain->lock, flags);
-        *dma_addr = __map_single(dev, domain->priv, paddr,
+        *dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
                                 size, DMA_BIDIRECTIONAL, true, dma_mask);
        if (*dma_addr == DMA_ERROR_CODE) {
@@ -2966,11 +2959,12 @@ static void *alloc_coherent(struct device *dev, size_t size,
        spin_unlock_irqrestore(&domain->lock, flags);
-        return virt_addr;
+        return page_address(page);
 out_free:
-        free_pages((unsigned long)virt_addr, get_order(size));
+        if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
+                __free_pages(page, get_order(size));
        return NULL;
 }
@@ -2982,11 +2976,15 @@ static void free_coherent(struct device *dev, size_t size,
                          void *virt_addr, dma_addr_t dma_addr,
                          struct dma_attrs *attrs)
 {
-        unsigned long flags;
        struct protection_domain *domain;
+        unsigned long flags;
+        struct page *page;
        INC_STATS_COUNTER(cnt_free_coherent);
+        page = virt_to_page(virt_addr);
+        size = PAGE_ALIGN(size);
        domain = get_domain(dev);
        if (IS_ERR(domain))
                goto free_mem;
@@ -3000,7 +2998,8 @@ static void free_coherent(struct device *dev, size_t size,
        spin_unlock_irqrestore(&domain->lock, flags);
 free_mem:
-        free_pages((unsigned long)virt_addr, get_order(size));
+        if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
+                __free_pages(page, get_order(size));
 }
 /*
@@ -3236,42 +3235,45 @@ static int __init alloc_passthrough_domain(void)
        return 0;
 }
-static int amd_iommu_domain_init(struct iommu_domain *dom)
+static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
 {
-        struct protection_domain *domain;
+        struct protection_domain *pdomain;
-        domain = protection_domain_alloc();
+        /* We only support unmanaged domains for now */
-        if (!domain)
+        if (type != IOMMU_DOMAIN_UNMANAGED)
-                goto out_free;
+                return NULL;
-        domain->mode    = PAGE_MODE_3_LEVEL;
+        pdomain = protection_domain_alloc();
-        domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
+        if (!pdomain)
-        if (!domain->pt_root)
                goto out_free;
-        domain->iommu_domain = dom;
+        pdomain->mode    = PAGE_MODE_3_LEVEL;
+        pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
-        dom->priv = domain;
+        if (!pdomain->pt_root)
+                goto out_free;
-        dom->geometry.aperture_start = 0;
+        pdomain->domain.geometry.aperture_start = 0;
-        dom->geometry.aperture_end   = ~0ULL;
+        pdomain->domain.geometry.aperture_end   = ~0ULL;
-        dom->geometry.force_aperture = true;
+        pdomain->domain.geometry.force_aperture = true;
-        return 0;
+        return &pdomain->domain;
 out_free:
-        protection_domain_free(domain);
+        protection_domain_free(pdomain);
-        return -ENOMEM;
+        return NULL;
 }
-static void amd_iommu_domain_destroy(struct iommu_domain *dom)
+static void amd_iommu_domain_free(struct iommu_domain *dom)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain;
-        if (!domain)
+        if (!dom)
                return;
+        domain = to_pdomain(dom);
        if (domain->dev_cnt > 0)
                cleanup_domain(domain);
@@ -3284,8 +3286,6 @@ static void amd_iommu_domain_destroy(struct iommu_domain *dom)
                free_gcr3_table(domain);
        protection_domain_free(domain);
-        dom->priv = NULL;
 }
 static void amd_iommu_detach_device(struct iommu_domain *dom,
@@ -3313,7 +3313,7 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
 static int amd_iommu_attach_device(struct iommu_domain *dom,
                                   struct device *dev)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        struct iommu_dev_data *dev_data;
        struct amd_iommu *iommu;
        int ret;
@@ -3340,7 +3340,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
                         phys_addr_t paddr, size_t page_size, int iommu_prot)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        int prot = 0;
        int ret;
@@ -3362,7 +3362,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
 static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
                           size_t page_size)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        size_t unmap_size;
        if (domain->mode == PAGE_MODE_NONE)
@@ -3380,28 +3380,22 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
                                          dma_addr_t iova)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
-        unsigned long offset_mask;
+        unsigned long offset_mask, pte_pgsize;
-        phys_addr_t paddr;
        u64 *pte, __pte;
        if (domain->mode == PAGE_MODE_NONE)
                return iova;
-        pte = fetch_pte(domain, iova);
+        pte = fetch_pte(domain, iova, &pte_pgsize);
        if (!pte || !IOMMU_PTE_PRESENT(*pte))
                return 0;
-        if (PM_PTE_LEVEL(*pte) == 0)
+        offset_mask = pte_pgsize - 1;
-                offset_mask = PAGE_SIZE - 1;
+        __pte       = *pte & PM_ADDR_MASK;
-        else
-                offset_mask = PTE_PAGE_SIZE(*pte) - 1;
-        __pte = *pte & PM_ADDR_MASK;
-        paddr = (__pte & ~offset_mask) | (iova & offset_mask);
-        return paddr;
+        return (__pte & ~offset_mask) | (iova & offset_mask);
 }
 static bool amd_iommu_capable(enum iommu_cap cap)
@@ -3420,8 +3414,8 @@ static bool amd_iommu_capable(enum iommu_cap cap)
 static const struct iommu_ops amd_iommu_ops = {
        .capable = amd_iommu_capable,
-        .domain_init = amd_iommu_domain_init,
+        .domain_alloc = amd_iommu_domain_alloc,
-        .domain_destroy = amd_iommu_domain_destroy,
+        .domain_free  = amd_iommu_domain_free,
        .attach_dev = amd_iommu_attach_device,
        .detach_dev = amd_iommu_detach_device,
        .map = amd_iommu_map,
@@ -3483,7 +3477,7 @@ EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
 void amd_iommu_domain_direct_map(struct iommu_domain *dom)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        unsigned long flags;
        spin_lock_irqsave(&domain->lock, flags);
@@ -3504,7 +3498,7 @@ EXPORT_SYMBOL(amd_iommu_domain_direct_map);
 int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        unsigned long flags;
        int levels, ret;
@@ -3616,7 +3610,7 @@ static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
 int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
                         u64 address)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        unsigned long flags;
        int ret;
@@ -3638,7 +3632,7 @@ static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
 int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        unsigned long flags;
        int ret;
@@ -3718,7 +3712,7 @@ static int __clear_gcr3(struct protection_domain *domain, int pasid)
 int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
                              unsigned long cr3)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        unsigned long flags;
        int ret;
@@ -3732,7 +3726,7 @@ EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
 int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
 {
-        struct protection_domain *domain = dom->priv;
+        struct protection_domain *domain = to_pdomain(dom);
        unsigned long flags;
        int ret;
@@ -3765,17 +3759,17 @@ EXPORT_SYMBOL(amd_iommu_complete_ppr);
 struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
 {
-        struct protection_domain *domain;
+        struct protection_domain *pdomain;
-        domain = get_domain(&pdev->dev);
+        pdomain = get_domain(&pdev->dev);
-        if (IS_ERR(domain))
+        if (IS_ERR(pdomain))
                return NULL;
        /* Only return IOMMUv2 domains */
-        if (!(domain->flags & PD_IOMMUV2_MASK))
+        if (!(pdomain->flags & PD_IOMMUV2_MASK))
                return NULL;
-        return domain->iommu_domain;
+        return &pdomain->domain;
 }
 EXPORT_SYMBOL(amd_iommu_get_v2_domain);
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index c4fffb710c58..05030e523771 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -282,6 +282,12 @@
 #define PTE_PAGE_SIZE(pte) \
        (1ULL << (1 + ffz(((pte) | 0xfffULL))))
+/*
+ * Takes a page-table level and returns the default page-size for this level
+ */
+#define PTE_LEVEL_PAGE_SIZE(level)                      \
+        (1ULL << (12 + (9 * (level))))
 #define IOMMU_PTE_P  (1ULL << 0)
 #define IOMMU_PTE_TV (1ULL << 1)
 #define IOMMU_PTE_U  (1ULL << 59)
@@ -400,6 +406,8 @@ struct iommu_domain;
 struct protection_domain {
        struct list_head list;  /* for list of all protection domains */
        struct list_head dev_list; /* List of all devices in this domain */
+        struct iommu_domain domain; /* generic domain handle used by
+                                       iommu core code */
        spinlock_t lock;        /* mostly used to lock the page table*/
        struct mutex api_lock;  /* protect page tables in the iommu-api path */
        u16 id;                 /* the domain id written to the device table */
@@ -411,10 +419,7 @@ struct protection_domain {
        bool updated;           /* complete domain flush required */
        unsigned dev_cnt;       /* devices assigned to this domain */
        unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
-        void *priv;             /* private data */
+        void *priv;             /* private data */
-        struct iommu_domain *iommu_domain; /* Pointer to generic
-                                              domain structure */
 };
 /*
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 6d5a5c44453b..a1cbba9056fd 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -417,7 +417,7 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
        dev_state      = pasid_state->device_state;
        run_inv_ctx_cb = !pasid_state->invalid;
-        if (run_inv_ctx_cb && pasid_state->device_state->inv_ctx_cb)
+        if (run_inv_ctx_cb && dev_state->inv_ctx_cb)
                dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);
        unbind_pasid(pasid_state);
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index a3adde6519f0..9f7e1d34a32b 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -343,6 +343,7 @@ struct arm_smmu_domain {
        struct arm_smmu_cfg             cfg;
        enum arm_smmu_domain_stage      stage;
        struct mutex                    init_mutex; /* Protects smmu pointer */
+        struct iommu_domain             domain;
 };
 static struct iommu_ops arm_smmu_ops;
@@ -360,6 +361,11 @@ static struct arm_smmu_option_prop arm_smmu_options[] = {
        { 0, NULL},
 };
+static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct arm_smmu_domain, domain);
+}
 static void parse_driver_options(struct arm_smmu_device *smmu)
 {
        int i = 0;
@@ -645,7 +651,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
        u32 fsr, far, fsynr, resume;
        unsigned long iova;
        struct iommu_domain *domain = dev;
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
        struct arm_smmu_device *smmu = smmu_domain->smmu;
        void __iomem *cb_base;
@@ -730,6 +736,20 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
        stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
        cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+        if (smmu->version > ARM_SMMU_V1) {
+                /*
+                 * CBA2R.
+                 * *Must* be initialised before CBAR thanks to VMID16
+                 * architectural oversight affected some implementations.
+                 */
+#ifdef CONFIG_64BIT
+                reg = CBA2R_RW64_64BIT;
+#else
+                reg = CBA2R_RW64_32BIT;
+#endif
+                writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
+        }
        /* CBAR */
        reg = cfg->cbar;
        if (smmu->version == ARM_SMMU_V1)
@@ -747,16 +767,6 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
        }
        writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
-        if (smmu->version > ARM_SMMU_V1) {
-                /* CBA2R */
-#ifdef CONFIG_64BIT
-                reg = CBA2R_RW64_64BIT;
-#else
-                reg = CBA2R_RW64_32BIT;
-#endif
-                writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
-        }
        /* TTBRs */
        if (stage1) {
                reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
@@ -836,7 +846,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
        struct io_pgtable_ops *pgtbl_ops;
        struct io_pgtable_cfg pgtbl_cfg;
        enum io_pgtable_fmt fmt;
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
        mutex_lock(&smmu_domain->init_mutex);
@@ -958,7 +968,7 @@ out_unlock:
 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 {
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_device *smmu = smmu_domain->smmu;
        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
        void __iomem *cb_base;
@@ -985,10 +995,12 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
        __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 }
-static int arm_smmu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 {
        struct arm_smmu_domain *smmu_domain;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
        /*
         * Allocate the domain and initialise some of its data structures.
         * We can't really do anything meaningful until we've added a
@@ -996,17 +1008,17 @@ static int arm_smmu_domain_init(struct iommu_domain *domain)
         */
        smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
        if (!smmu_domain)
-                return -ENOMEM;
+                return NULL;
        mutex_init(&smmu_domain->init_mutex);
        spin_lock_init(&smmu_domain->pgtbl_lock);
-        domain->priv = smmu_domain;
-        return 0;
+        return &smmu_domain->domain;
 }
-static void arm_smmu_domain_destroy(struct iommu_domain *domain)
+static void arm_smmu_domain_free(struct iommu_domain *domain)
 {
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        /*
         * Free the domain resources. We assume that all devices have
@@ -1143,7 +1155,7 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 {
        int ret;
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_device *smmu;
        struct arm_smmu_master_cfg *cfg;
@@ -1187,7 +1199,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
 {
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_master_cfg *cfg;
        cfg = find_smmu_master_cfg(dev);
@@ -1203,7 +1215,7 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
 {
        int ret;
        unsigned long flags;
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
        if (!ops)
@@ -1220,7 +1232,7 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
 {
        size_t ret;
        unsigned long flags;
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
        if (!ops)
@@ -1235,7 +1247,7 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
                                              dma_addr_t iova)
 {
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_device *smmu = smmu_domain->smmu;
        struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
        struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
@@ -1281,7 +1293,7 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
 {
        phys_addr_t ret;
        unsigned long flags;
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
        if (!ops)
@@ -1329,61 +1341,83 @@ static void __arm_smmu_release_pci_iommudata(void *data)
        kfree(data);
 }
-static int arm_smmu_add_device(struct device *dev)
+static int arm_smmu_add_pci_device(struct pci_dev *pdev)
 {
-        struct arm_smmu_device *smmu;
+        int i, ret;
-        struct arm_smmu_master_cfg *cfg;
+        u16 sid;
        struct iommu_group *group;
-        void (*releasefn)(void *) = NULL;
+        struct arm_smmu_master_cfg *cfg;
-        int ret;
-        smmu = find_smmu_for_device(dev);
-        if (!smmu)
-                return -ENODEV;
-        group = iommu_group_alloc();
+        group = iommu_group_get_for_dev(&pdev->dev);
-        if (IS_ERR(group)) {
+        if (IS_ERR(group))
-                dev_err(dev, "Failed to allocate IOMMU group\n");
                return PTR_ERR(group);
-        }
-        if (dev_is_pci(dev)) {
-                struct pci_dev *pdev = to_pci_dev(dev);
+        cfg = iommu_group_get_iommudata(group);
+        if (!cfg) {
                cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
                if (!cfg) {
                        ret = -ENOMEM;
                        goto out_put_group;
                }
-                cfg->num_streamids = 1;
+                iommu_group_set_iommudata(group, cfg,
-                /*
+                                          __arm_smmu_release_pci_iommudata);
-                 * Assume Stream ID == Requester ID for now.
+        }
-                 * We need a way to describe the ID mappings in FDT.
-                 */
-                pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid,
-                                       &cfg->streamids[0]);
-                releasefn = __arm_smmu_release_pci_iommudata;
-        } else {
-                struct arm_smmu_master *master;
-                master = find_smmu_master(smmu, dev->of_node);
-                if (!master) {
-                        ret = -ENODEV;
-                        goto out_put_group;
-                }
-                cfg = &master->cfg;
+        if (cfg->num_streamids >= MAX_MASTER_STREAMIDS) {
+                ret = -ENOSPC;
+                goto out_put_group;
        }
-        iommu_group_set_iommudata(group, cfg, releasefn);
+        /*
-        ret = iommu_group_add_device(group, dev);
+         * Assume Stream ID == Requester ID for now.
+         * We need a way to describe the ID mappings in FDT.
+         */
+        pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
+        for (i = 0; i < cfg->num_streamids; ++i)
+                if (cfg->streamids[i] == sid)
+                        break;
+        /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
+        if (i == cfg->num_streamids)
+                cfg->streamids[cfg->num_streamids++] = sid;
+        return 0;
 out_put_group:
        iommu_group_put(group);
        return ret;
 }
+static int arm_smmu_add_platform_device(struct device *dev)
+{
+        struct iommu_group *group;
+        struct arm_smmu_master *master;
+        struct arm_smmu_device *smmu = find_smmu_for_device(dev);
+        if (!smmu)
+                return -ENODEV;
+        master = find_smmu_master(smmu, dev->of_node);
+        if (!master)
+                return -ENODEV;
+        /* No automatic group creation for platform devices */
+        group = iommu_group_alloc();
+        if (IS_ERR(group))
+                return PTR_ERR(group);
+        iommu_group_set_iommudata(group, &master->cfg, NULL);
+        return iommu_group_add_device(group, dev);
+}
+static int arm_smmu_add_device(struct device *dev)
+{
+        if (dev_is_pci(dev))
+                return arm_smmu_add_pci_device(to_pci_dev(dev));
+        return arm_smmu_add_platform_device(dev);
+}
 static void arm_smmu_remove_device(struct device *dev)
 {
        iommu_group_remove_device(dev);
@@ -1392,7 +1426,7 @@ static void arm_smmu_remove_device(struct device *dev)
 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
                                    enum iommu_attr attr, void *data)
 {
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        switch (attr) {
        case DOMAIN_ATTR_NESTING:
@@ -1407,7 +1441,7 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
                                    enum iommu_attr attr, void *data)
 {
        int ret = 0;
-        struct arm_smmu_domain *smmu_domain = domain->priv;
+        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        mutex_lock(&smmu_domain->init_mutex);
@@ -1435,8 +1469,8 @@ out_unlock:
 static struct iommu_ops arm_smmu_ops = {
        .capable                = arm_smmu_capable,
-        .domain_init            = arm_smmu_domain_init,
+        .domain_alloc           = arm_smmu_domain_alloc,
-        .domain_destroy         = arm_smmu_domain_destroy,
+        .domain_free            = arm_smmu_domain_free,
        .attach_dev             = arm_smmu_attach_dev,
        .detach_dev             = arm_smmu_detach_dev,
        .map                    = arm_smmu_map,
@@ -1633,6 +1667,15 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
        size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
        smmu->pa_size = size;
+        /*
+         * What the page table walker can address actually depends on which
+         * descriptor format is in use, but since a) we don't know that yet,
+         * and b) it can vary per context bank, this will have to do...
+         */
+        if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
+                dev_warn(smmu->dev,
+                         "failed to set DMA mask for table walker\n");
        if (smmu->version == ARM_SMMU_V1) {
                smmu->va_size = smmu->ipa_size;
                size = SZ_4K | SZ_2M | SZ_1G;
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index dc14fec4ede1..3e898504a7c4 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -200,6 +200,7 @@ struct exynos_iommu_domain {
        short *lv2entcnt; /* free lv2 entry counter for each section */
        spinlock_t lock; /* lock for this structure */
        spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
+        struct iommu_domain domain; /* generic domain data structure */
 };
 struct sysmmu_drvdata {
@@ -214,6 +215,11 @@ struct sysmmu_drvdata {
        phys_addr_t pgtable;
 };
+static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct exynos_iommu_domain, domain);
+}
 static bool set_sysmmu_active(struct sysmmu_drvdata *data)
 {
        /* return true if the System MMU was not active previously
@@ -696,58 +702,60 @@ static inline void pgtable_flush(void *vastart, void *vaend)
                                virt_to_phys(vaend));
 }
-static int exynos_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
 {
-        struct exynos_iommu_domain *priv;
+        struct exynos_iommu_domain *exynos_domain;
        int i;
-        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        if (type != IOMMU_DOMAIN_UNMANAGED)
-        if (!priv)
+                return NULL;
-                return -ENOMEM;
+        exynos_domain = kzalloc(sizeof(*exynos_domain), GFP_KERNEL);
+        if (!exynos_domain)
+                return NULL;
-        priv->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
+        exynos_domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
-        if (!priv->pgtable)
+        if (!exynos_domain->pgtable)
                goto err_pgtable;
-        priv->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
+        exynos_domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
-        if (!priv->lv2entcnt)
+        if (!exynos_domain->lv2entcnt)
                goto err_counter;
        /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
        for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
-                priv->pgtable[i + 0] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 0] = ZERO_LV2LINK;
-                priv->pgtable[i + 1] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 1] = ZERO_LV2LINK;
-                priv->pgtable[i + 2] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 2] = ZERO_LV2LINK;
-                priv->pgtable[i + 3] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 3] = ZERO_LV2LINK;
-                priv->pgtable[i + 4] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 4] = ZERO_LV2LINK;
-                priv->pgtable[i + 5] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 5] = ZERO_LV2LINK;
-                priv->pgtable[i + 6] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 6] = ZERO_LV2LINK;
-                priv->pgtable[i + 7] = ZERO_LV2LINK;
+                exynos_domain->pgtable[i + 7] = ZERO_LV2LINK;
        }
-        pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES);
+        pgtable_flush(exynos_domain->pgtable, exynos_domain->pgtable + NUM_LV1ENTRIES);
-        spin_lock_init(&priv->lock);
+        spin_lock_init(&exynos_domain->lock);
-        spin_lock_init(&priv->pgtablelock);
+        spin_lock_init(&exynos_domain->pgtablelock);
-        INIT_LIST_HEAD(&priv->clients);
+        INIT_LIST_HEAD(&exynos_domain->clients);
-        domain->geometry.aperture_start = 0;
+        exynos_domain->domain.geometry.aperture_start = 0;
-        domain->geometry.aperture_end   = ~0UL;
+        exynos_domain->domain.geometry.aperture_end   = ~0UL;
-        domain->geometry.force_aperture = true;
+        exynos_domain->domain.geometry.force_aperture = true;
-        domain->priv = priv;
+        return &exynos_domain->domain;
-        return 0;
 err_counter:
-        free_pages((unsigned long)priv->pgtable, 2);
+        free_pages((unsigned long)exynos_domain->pgtable, 2);
 err_pgtable:
-        kfree(priv);
+        kfree(exynos_domain);
-        return -ENOMEM;
+        return NULL;
 }
-static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
+static void exynos_iommu_domain_free(struct iommu_domain *domain)
 {
-        struct exynos_iommu_domain *priv = domain->priv;
+        struct exynos_iommu_domain *priv = to_exynos_domain(domain);
        struct exynos_iommu_owner *owner;
        unsigned long flags;
        int i;
@@ -773,15 +781,14 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
        free_pages((unsigned long)priv->pgtable, 2);
        free_pages((unsigned long)priv->lv2entcnt, 1);
-        kfree(domain->priv);
+        kfree(priv);
-        domain->priv = NULL;
 }
 static int exynos_iommu_attach_device(struct iommu_domain *domain,
                                   struct device *dev)
 {
        struct exynos_iommu_owner *owner = dev->archdata.iommu;
-        struct exynos_iommu_domain *priv = domain->priv;
+        struct exynos_iommu_domain *priv = to_exynos_domain(domain);
        phys_addr_t pagetable = virt_to_phys(priv->pgtable);
        unsigned long flags;
        int ret;
@@ -812,7 +819,7 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain,
                                    struct device *dev)
 {
        struct exynos_iommu_owner *owner;
-        struct exynos_iommu_domain *priv = domain->priv;
+        struct exynos_iommu_domain *priv = to_exynos_domain(domain);
        phys_addr_t pagetable = virt_to_phys(priv->pgtable);
        unsigned long flags;
@@ -988,7 +995,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
 static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova,
                         phys_addr_t paddr, size_t size, int prot)
 {
-        struct exynos_iommu_domain *priv = domain->priv;
+        struct exynos_iommu_domain *priv = to_exynos_domain(domain);
        sysmmu_pte_t *entry;
        sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
        unsigned long flags;
@@ -1042,7 +1049,7 @@ static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv,
 static size_t exynos_iommu_unmap(struct iommu_domain *domain,
                                        unsigned long l_iova, size_t size)
 {
-        struct exynos_iommu_domain *priv = domain->priv;
+        struct exynos_iommu_domain *priv = to_exynos_domain(domain);
        sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
        sysmmu_pte_t *ent;
        size_t err_pgsize;
@@ -1119,7 +1126,7 @@ err:
 static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
                                          dma_addr_t iova)
 {
-        struct exynos_iommu_domain *priv = domain->priv;
+        struct exynos_iommu_domain *priv = to_exynos_domain(domain);
        sysmmu_pte_t *entry;
        unsigned long flags;
        phys_addr_t phys = 0;
@@ -1171,8 +1178,8 @@ static void exynos_iommu_remove_device(struct device *dev)
 }
 static const struct iommu_ops exynos_iommu_ops = {
-        .domain_init = exynos_iommu_domain_init,
+        .domain_alloc = exynos_iommu_domain_alloc,
-        .domain_destroy = exynos_iommu_domain_destroy,
+        .domain_free = exynos_iommu_domain_free,
        .attach_dev = exynos_iommu_attach_device,
        .detach_dev = exynos_iommu_detach_device,
        .map = exynos_iommu_map,
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index ceebd287b660..1d452930c890 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -33,6 +33,11 @@ static struct kmem_cache *fsl_pamu_domain_cache;
 static struct kmem_cache *iommu_devinfo_cache;
 static DEFINE_SPINLOCK(device_domain_lock);
+static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct fsl_dma_domain, iommu_domain);
+}
 static int __init iommu_init_mempool(void)
 {
        fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
@@ -65,7 +70,7 @@ static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t i
        struct dma_window *win_ptr = &dma_domain->win_arr[0];
        struct iommu_domain_geometry *geom;
-        geom = &dma_domain->iommu_domain->geometry;
+        geom = &dma_domain->iommu_domain.geometry;
        if (!win_cnt || !dma_domain->geom_size) {
                pr_debug("Number of windows/geometry not configured for the domain\n");
@@ -123,7 +128,7 @@ static int map_win(int liodn, struct fsl_dma_domain *dma_domain)
 {
        int ret;
        struct dma_window *wnd = &dma_domain->win_arr[0];
-        phys_addr_t wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
+        phys_addr_t wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
        unsigned long flags;
        spin_lock_irqsave(&iommu_lock, flags);
@@ -172,7 +177,7 @@ static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr
        } else {
                phys_addr_t wnd_addr;
-                wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
+                wnd_addr = dma_domain->iommu_domain.geometry.aperture_start;
                ret = pamu_config_ppaace(liodn, wnd_addr,
                                         wnd->size,
@@ -384,7 +389,7 @@ static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct d
 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
                                         dma_addr_t iova)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        if (iova < domain->geometry.aperture_start ||
            iova > domain->geometry.aperture_end)
@@ -398,11 +403,9 @@ static bool fsl_pamu_capable(enum iommu_cap cap)
        return cap == IOMMU_CAP_CACHE_COHERENCY;
 }
-static void fsl_pamu_domain_destroy(struct iommu_domain *domain)
+static void fsl_pamu_domain_free(struct iommu_domain *domain)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
-        domain->priv = NULL;
        /* remove all the devices from the device list */
        detach_device(NULL, dma_domain);
@@ -413,23 +416,24 @@ static void fsl_pamu_domain_destroy(struct iommu_domain *domain)
        kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
 }
-static int fsl_pamu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
 {
        struct fsl_dma_domain *dma_domain;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
        dma_domain = iommu_alloc_dma_domain();
        if (!dma_domain) {
                pr_debug("dma_domain allocation failed\n");
-                return -ENOMEM;
+                return NULL;
        }
-        domain->priv = dma_domain;
-        dma_domain->iommu_domain = domain;
        /* defaul geometry 64 GB i.e. maximum system address */
-        domain->geometry.aperture_start = 0;
+        dma_domain->iommu_domain. geometry.aperture_start = 0;
-        domain->geometry.aperture_end = (1ULL << 36) - 1;
+        dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
-        domain->geometry.force_aperture = true;
+        dma_domain->iommu_domain.geometry.force_aperture = true;
-        return 0;
+        return &dma_domain->iommu_domain;
 }
 /* Configure geometry settings for all LIODNs associated with domain */
@@ -499,7 +503,7 @@ static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        unsigned long flags;
        int ret;
@@ -530,7 +534,7 @@ static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
 static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr,
                                  phys_addr_t paddr, u64 size, int prot)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        struct dma_window *wnd;
        int pamu_prot = 0;
        int ret;
@@ -607,7 +611,7 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain,
                                int num)
 {
        unsigned long flags;
-        struct iommu_domain *domain = dma_domain->iommu_domain;
+        struct iommu_domain *domain = &dma_domain->iommu_domain;
        int ret = 0;
        int i;
@@ -653,7 +657,7 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain,
 static int fsl_pamu_attach_device(struct iommu_domain *domain,
                                  struct device *dev)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        const u32 *liodn;
        u32 liodn_cnt;
        int len, ret = 0;
@@ -691,7 +695,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
 static void fsl_pamu_detach_device(struct iommu_domain *domain,
                                   struct device *dev)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        const u32 *prop;
        int len;
        struct pci_dev *pdev = NULL;
@@ -723,7 +727,7 @@ static void fsl_pamu_detach_device(struct iommu_domain *domain,
 static  int configure_domain_geometry(struct iommu_domain *domain, void *data)
 {
        struct iommu_domain_geometry *geom_attr = data;
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        dma_addr_t geom_size;
        unsigned long flags;
@@ -813,7 +817,7 @@ static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool en
 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
                                    enum iommu_attr attr_type, void *data)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        int ret = 0;
        switch (attr_type) {
@@ -838,7 +842,7 @@ static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
 static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
                                    enum iommu_attr attr_type, void *data)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        int ret = 0;
        switch (attr_type) {
@@ -999,7 +1003,7 @@ static void fsl_pamu_remove_device(struct device *dev)
 static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        unsigned long flags;
        int ret;
@@ -1048,15 +1052,15 @@ static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
 static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
 {
-        struct fsl_dma_domain *dma_domain = domain->priv;
+        struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
        return dma_domain->win_cnt;
 }
 static const struct iommu_ops fsl_pamu_ops = {
        .capable        = fsl_pamu_capable,
-        .domain_init    = fsl_pamu_domain_init,
+        .domain_alloc   = fsl_pamu_domain_alloc,
-        .domain_destroy = fsl_pamu_domain_destroy,
+        .domain_free    = fsl_pamu_domain_free,
        .attach_dev     = fsl_pamu_attach_device,
        .detach_dev     = fsl_pamu_detach_device,
        .domain_window_enable = fsl_pamu_window_enable,
diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h
index c90293f99709..f2b0f741d3de 100644
--- a/drivers/iommu/fsl_pamu_domain.h
+++ b/drivers/iommu/fsl_pamu_domain.h
@@ -71,7 +71,7 @@ struct fsl_dma_domain {
        u32                             stash_id;
        struct pamu_stash_attribute     dma_stash;
        u32                             snoop_id;
-        struct iommu_domain             *iommu_domain;
+        struct iommu_domain             iommu_domain;
        spinlock_t                      domain_lock;
 };
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 4fc1f8a7f98e..a35927cd42e5 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -339,7 +339,7 @@ struct dmar_domain {
        DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
                                        /* bitmap of iommus this domain uses*/
-        struct list_head devices;       /* all devices' list */
+        struct list_head devices;       /* all devices' list */
        struct iova_domain iovad;       /* iova's that belong to this domain */
        struct dma_pte  *pgd;           /* virtual address */
@@ -358,6 +358,9 @@ struct dmar_domain {
                                           2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
        spinlock_t      iommu_lock;     /* protect iommu set in domain */
        u64             max_addr;       /* maximum mapped address */
+        struct iommu_domain domain;     /* generic domain data structure for
+                                           iommu core */
 };
 /* PCI domain-device relationship */
@@ -449,6 +452,12 @@ static LIST_HEAD(device_domain_list);
 static const struct iommu_ops intel_iommu_ops;
+/* Convert generic 'struct iommu_domain to private struct dmar_domain */
+static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct dmar_domain, domain);
+}
 static int __init intel_iommu_setup(char *str)
 {
        if (!str)
@@ -595,12 +604,13 @@ static void domain_update_iommu_coherency(struct dmar_domain *domain)
 {
        struct dmar_drhd_unit *drhd;
        struct intel_iommu *iommu;
-        int i, found = 0;
+        bool found = false;
+        int i;
        domain->iommu_coherency = 1;
        for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
-                found = 1;
+                found = true;
                if (!ecap_coherent(g_iommus[i]->ecap)) {
                        domain->iommu_coherency = 0;
                        break;
@@ -1267,7 +1277,7 @@ static struct device_domain_info *
 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
                         u8 bus, u8 devfn)
 {
-        int found = 0;
+        bool found = false;
        unsigned long flags;
        struct device_domain_info *info;
        struct pci_dev *pdev;
@@ -1282,7 +1292,7 @@ iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
        list_for_each_entry(info, &domain->devices, link)
                if (info->iommu == iommu && info->bus == bus &&
                    info->devfn == devfn) {
-                        found = 1;
+                        found = true;
                        break;
                }
        spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -4269,7 +4279,7 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
        struct device_domain_info *info, *tmp;
        struct intel_iommu *iommu;
        unsigned long flags;
-        int found = 0;
+        bool found = false;
        u8 bus, devfn;
        iommu = device_to_iommu(dev, &bus, &devfn);
@@ -4301,7 +4311,7 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
                 * update iommu count and coherency
                 */
                if (info->iommu == iommu)
-                        found = 1;
+                        found = true;
        }
        spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -4339,44 +4349,45 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
        return 0;
 }
-static int intel_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
 {
        struct dmar_domain *dmar_domain;
+        struct iommu_domain *domain;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
        dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
        if (!dmar_domain) {
                printk(KERN_ERR
                        "intel_iommu_domain_init: dmar_domain == NULL\n");
-                return -ENOMEM;
+                return NULL;
        }
        if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
                printk(KERN_ERR
                        "intel_iommu_domain_init() failed\n");
                domain_exit(dmar_domain);
-                return -ENOMEM;
+                return NULL;
        }
        domain_update_iommu_cap(dmar_domain);
-        domain->priv = dmar_domain;
+        domain = &dmar_domain->domain;
        domain->geometry.aperture_start = 0;
        domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
        domain->geometry.force_aperture = true;
-        return 0;
+        return domain;
 }
-static void intel_iommu_domain_destroy(struct iommu_domain *domain)
+static void intel_iommu_domain_free(struct iommu_domain *domain)
 {
-        struct dmar_domain *dmar_domain = domain->priv;
+        domain_exit(to_dmar_domain(domain));
-        domain->priv = NULL;
-        domain_exit(dmar_domain);
 }
 static int intel_iommu_attach_device(struct iommu_domain *domain,
                                     struct device *dev)
 {
-        struct dmar_domain *dmar_domain = domain->priv;
+        struct dmar_domain *dmar_domain = to_dmar_domain(domain);
        struct intel_iommu *iommu;
        int addr_width;
        u8 bus, devfn;
@@ -4441,16 +4452,14 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
 static void intel_iommu_detach_device(struct iommu_domain *domain,
                                      struct device *dev)
 {
-        struct dmar_domain *dmar_domain = domain->priv;
+        domain_remove_one_dev_info(to_dmar_domain(domain), dev);
-        domain_remove_one_dev_info(dmar_domain, dev);
 }
 static int intel_iommu_map(struct iommu_domain *domain,
                           unsigned long iova, phys_addr_t hpa,
                           size_t size, int iommu_prot)
 {
-        struct dmar_domain *dmar_domain = domain->priv;
+        struct dmar_domain *dmar_domain = to_dmar_domain(domain);
        u64 max_addr;
        int prot = 0;
        int ret;
@@ -4487,7 +4496,7 @@ static int intel_iommu_map(struct iommu_domain *domain,
 static size_t intel_iommu_unmap(struct iommu_domain *domain,
                                unsigned long iova, size_t size)
 {
-        struct dmar_domain *dmar_domain = domain->priv;
+        struct dmar_domain *dmar_domain = to_dmar_domain(domain);
        struct page *freelist = NULL;
        struct intel_iommu *iommu;
        unsigned long start_pfn, last_pfn;
@@ -4535,7 +4544,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
                                            dma_addr_t iova)
 {
-        struct dmar_domain *dmar_domain = domain->priv;
+        struct dmar_domain *dmar_domain = to_dmar_domain(domain);
        struct dma_pte *pte;
        int level = 0;
        u64 phys = 0;
@@ -4594,8 +4603,8 @@ static void intel_iommu_remove_device(struct device *dev)
 static const struct iommu_ops intel_iommu_ops = {
        .capable        = intel_iommu_capable,
-        .domain_init    = intel_iommu_domain_init,
+        .domain_alloc   = intel_iommu_domain_alloc,
-        .domain_destroy = intel_iommu_domain_destroy,
+        .domain_free    = intel_iommu_domain_free,
        .attach_dev     = intel_iommu_attach_device,
        .detach_dev     = intel_iommu_detach_device,
        .map            = intel_iommu_map,
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index 14de1ab223c8..6c25b3c5b729 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -631,7 +631,7 @@ static int __init intel_enable_irq_remapping(void)
 {
        struct dmar_drhd_unit *drhd;
        struct intel_iommu *iommu;
-        int setup = 0;
+        bool setup = false;
        int eim = 0;
        if (x2apic_supported()) {
@@ -697,7 +697,7 @@ static int __init intel_enable_irq_remapping(void)
         */
        for_each_iommu(iommu, drhd) {
                iommu_set_irq_remapping(iommu, eim);
-                setup = 1;
+                setup = true;
        }
        if (!setup)
@@ -856,7 +856,7 @@ static int __init parse_ioapics_under_ir(void)
 {
        struct dmar_drhd_unit *drhd;
        struct intel_iommu *iommu;
-        int ir_supported = 0;
+        bool ir_supported = false;
        int ioapic_idx;
        for_each_iommu(iommu, drhd)
@@ -864,7 +864,7 @@ static int __init parse_ioapics_under_ir(void)
                        if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
                                return -1;
-                        ir_supported = 1;
+                        ir_supported = true;
                }
        if (!ir_supported)
@@ -917,7 +917,7 @@ static void disable_irq_remapping(void)
 static int reenable_irq_remapping(int eim)
 {
        struct dmar_drhd_unit *drhd;
-        int setup = 0;
+        bool setup = false;
        struct intel_iommu *iommu = NULL;
        for_each_iommu(iommu, drhd)
@@ -933,7 +933,7 @@ static int reenable_irq_remapping(int eim)
                /* Set up interrupt remapping for iommu.*/
                iommu_set_irq_remapping(iommu, eim);
-                setup = 1;
+                setup = true;
        }
        if (!setup)
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index b610a8dee238..4e460216bd16 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -116,6 +116,8 @@
 #define ARM_32_LPAE_TCR_EAE             (1 << 31)
 #define ARM_64_LPAE_S2_TCR_RES1         (1 << 31)
+#define ARM_LPAE_TCR_EPD1               (1 << 23)
 #define ARM_LPAE_TCR_TG0_4K             (0 << 14)
 #define ARM_LPAE_TCR_TG0_64K            (1 << 14)
 #define ARM_LPAE_TCR_TG0_16K            (2 << 14)
@@ -621,6 +623,9 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
        }
        reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT;
+        /* Disable speculative walks through TTBR1 */
+        reg |= ARM_LPAE_TCR_EPD1;
        cfg->arm_lpae_s1_cfg.tcr = reg;
        /* MAIRs */
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 72e683df0731..d4f527e56679 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -901,36 +901,24 @@ EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
 struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
 {
        struct iommu_domain *domain;
-        int ret;
        if (bus == NULL || bus->iommu_ops == NULL)
                return NULL;
-        domain = kzalloc(sizeof(*domain), GFP_KERNEL);
+        domain = bus->iommu_ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED);
        if (!domain)
                return NULL;
-        domain->ops = bus->iommu_ops;
+        domain->ops  = bus->iommu_ops;
+        domain->type = IOMMU_DOMAIN_UNMANAGED;
-        ret = domain->ops->domain_init(domain);
-        if (ret)
-                goto out_free;
        return domain;
-out_free:
-        kfree(domain);
-        return NULL;
 }
 EXPORT_SYMBOL_GPL(iommu_domain_alloc);
 void iommu_domain_free(struct iommu_domain *domain)
 {
-        if (likely(domain->ops->domain_destroy != NULL))
+        domain->ops->domain_free(domain);
-                domain->ops->domain_destroy(domain);
-        kfree(domain);
 }
 EXPORT_SYMBOL_GPL(iommu_domain_free);
@@ -1049,6 +1037,9 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
                     domain->ops->pgsize_bitmap == 0UL))
                return -ENODEV;
+        if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
+                return -EINVAL;
        /* find out the minimum page size supported */
        min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
@@ -1100,6 +1091,9 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
                     domain->ops->pgsize_bitmap == 0UL))
                return -ENODEV;
+        if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
+                return -EINVAL;
        /* find out the minimum page size supported */
        min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index bc39bdf7b99b..1a67c531a07e 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -38,7 +38,7 @@ struct ipmmu_vmsa_device {
 struct ipmmu_vmsa_domain {
        struct ipmmu_vmsa_device *mmu;
-        struct iommu_domain *io_domain;
+        struct iommu_domain io_domain;
        struct io_pgtable_cfg cfg;
        struct io_pgtable_ops *iop;
@@ -56,6 +56,11 @@ struct ipmmu_vmsa_archdata {
 static DEFINE_SPINLOCK(ipmmu_devices_lock);
 static LIST_HEAD(ipmmu_devices);
+static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
+}
 #define TLB_LOOP_TIMEOUT                100     /* 100us */
 /* -----------------------------------------------------------------------------
@@ -428,7 +433,7 @@ static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)
         * TODO: We need to look up the faulty device based on the I/O VA. Use
         * the IOMMU device for now.
         */
-        if (!report_iommu_fault(domain->io_domain, mmu->dev, iova, 0))
+        if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))
                return IRQ_HANDLED;
        dev_err_ratelimited(mmu->dev,
@@ -448,7 +453,7 @@ static irqreturn_t ipmmu_irq(int irq, void *dev)
                return IRQ_NONE;
        io_domain = mmu->mapping->domain;
-        domain = io_domain->priv;
+        domain = to_vmsa_domain(io_domain);
        return ipmmu_domain_irq(domain);
 }
@@ -457,25 +462,25 @@ static irqreturn_t ipmmu_irq(int irq, void *dev)
 * IOMMU Operations
 */
-static int ipmmu_domain_init(struct iommu_domain *io_domain)
+static struct iommu_domain *ipmmu_domain_alloc(unsigned type)
 {
        struct ipmmu_vmsa_domain *domain;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
        domain = kzalloc(sizeof(*domain), GFP_KERNEL);
        if (!domain)
-                return -ENOMEM;
+                return NULL;
        spin_lock_init(&domain->lock);
-        io_domain->priv = domain;
+        return &domain->io_domain;
-        domain->io_domain = io_domain;
-        return 0;
 }
-static void ipmmu_domain_destroy(struct iommu_domain *io_domain)
+static void ipmmu_domain_free(struct iommu_domain *io_domain)
 {
-        struct ipmmu_vmsa_domain *domain = io_domain->priv;
+        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
        /*
         * Free the domain resources. We assume that all devices have already
@@ -491,7 +496,7 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
 {
        struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
        struct ipmmu_vmsa_device *mmu = archdata->mmu;
-        struct ipmmu_vmsa_domain *domain = io_domain->priv;
+        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
        unsigned long flags;
        unsigned int i;
        int ret = 0;
@@ -532,7 +537,7 @@ static void ipmmu_detach_device(struct iommu_domain *io_domain,
                                struct device *dev)
 {
        struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
-        struct ipmmu_vmsa_domain *domain = io_domain->priv;
+        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
        unsigned int i;
        for (i = 0; i < archdata->num_utlbs; ++i)
@@ -546,7 +551,7 @@ static void ipmmu_detach_device(struct iommu_domain *io_domain,
 static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
                     phys_addr_t paddr, size_t size, int prot)
 {
-        struct ipmmu_vmsa_domain *domain = io_domain->priv;
+        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
        if (!domain)
                return -ENODEV;
@@ -557,7 +562,7 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
 static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
                          size_t size)
 {
-        struct ipmmu_vmsa_domain *domain = io_domain->priv;
+        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
        return domain->iop->unmap(domain->iop, iova, size);
 }
@@ -565,7 +570,7 @@ static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
 static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
                                      dma_addr_t iova)
 {
-        struct ipmmu_vmsa_domain *domain = io_domain->priv;
+        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
        /* TODO: Is locking needed ? */
@@ -737,8 +742,8 @@ static void ipmmu_remove_device(struct device *dev)
 }
 static const struct iommu_ops ipmmu_ops = {
-        .domain_init = ipmmu_domain_init,
+        .domain_alloc = ipmmu_domain_alloc,
-        .domain_destroy = ipmmu_domain_destroy,
+        .domain_free = ipmmu_domain_free,
        .attach_dev = ipmmu_attach_device,
        .detach_dev = ipmmu_detach_device,
        .map = ipmmu_map,
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index e1b05379ca0e..15a2063812fa 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -52,8 +52,14 @@ DEFINE_SPINLOCK(msm_iommu_lock);
 struct msm_priv {
        unsigned long *pgtable;
        struct list_head list_attached;
+        struct iommu_domain domain;
 };
+static struct msm_priv *to_msm_priv(struct iommu_domain *dom)
+{
+        return container_of(dom, struct msm_priv, domain);
+}
 static int __enable_clocks(struct msm_iommu_drvdata *drvdata)
 {
        int ret;
@@ -79,7 +85,7 @@ static void __disable_clocks(struct msm_iommu_drvdata *drvdata)
 static int __flush_iotlb(struct iommu_domain *domain)
 {
-        struct msm_priv *priv = domain->priv;
+        struct msm_priv *priv = to_msm_priv(domain);
        struct msm_iommu_drvdata *iommu_drvdata;
        struct msm_iommu_ctx_drvdata *ctx_drvdata;
        int ret = 0;
@@ -209,10 +215,14 @@ static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable)
        SET_M(base, ctx, 1);
 }
-static int msm_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *msm_iommu_domain_alloc(unsigned type)
 {
-        struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+        struct msm_priv *priv;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
+        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
        if (!priv)
                goto fail_nomem;
@@ -224,20 +234,19 @@ static int msm_iommu_domain_init(struct iommu_domain *domain)
                goto fail_nomem;
        memset(priv->pgtable, 0, SZ_16K);
-        domain->priv = priv;
-        domain->geometry.aperture_start = 0;
+        priv->domain.geometry.aperture_start = 0;
-        domain->geometry.aperture_end   = (1ULL << 32) - 1;
+        priv->domain.geometry.aperture_end   = (1ULL << 32) - 1;
-        domain->geometry.force_aperture = true;
+        priv->domain.geometry.force_aperture = true;
-        return 0;
+        return &priv->domain;
 fail_nomem:
        kfree(priv);
-        return -ENOMEM;
+        return NULL;
 }
-static void msm_iommu_domain_destroy(struct iommu_domain *domain)
+static void msm_iommu_domain_free(struct iommu_domain *domain)
 {
        struct msm_priv *priv;
        unsigned long flags;
@@ -245,20 +254,17 @@ static void msm_iommu_domain_destroy(struct iommu_domain *domain)
        int i;
        spin_lock_irqsave(&msm_iommu_lock, flags);
-        priv = domain->priv;
+        priv = to_msm_priv(domain);
-        domain->priv = NULL;
-        if (priv) {
+        fl_table = priv->pgtable;
-                fl_table = priv->pgtable;
-                for (i = 0; i < NUM_FL_PTE; i++)
+        for (i = 0; i < NUM_FL_PTE; i++)
-                        if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
+                if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
-                                free_page((unsigned long) __va(((fl_table[i]) &
+                        free_page((unsigned long) __va(((fl_table[i]) &
-                                                                FL_BASE_MASK)));
+                                                        FL_BASE_MASK)));
-                free_pages((unsigned long)priv->pgtable, get_order(SZ_16K));
+        free_pages((unsigned long)priv->pgtable, get_order(SZ_16K));
-                priv->pgtable = NULL;
+        priv->pgtable = NULL;
-        }
        kfree(priv);
        spin_unlock_irqrestore(&msm_iommu_lock, flags);
@@ -276,9 +282,9 @@ static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
        spin_lock_irqsave(&msm_iommu_lock, flags);
-        priv = domain->priv;
+        priv = to_msm_priv(domain);
-        if (!priv || !dev) {
+        if (!dev) {
                ret = -EINVAL;
                goto fail;
        }
@@ -330,9 +336,9 @@ static void msm_iommu_detach_dev(struct iommu_domain *domain,
        int ret;
        spin_lock_irqsave(&msm_iommu_lock, flags);
-        priv = domain->priv;
+        priv = to_msm_priv(domain);
-        if (!priv || !dev)
+        if (!dev)
                goto fail;
        iommu_drvdata = dev_get_drvdata(dev->parent);
@@ -382,11 +388,7 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
                goto fail;
        }
-        priv = domain->priv;
+        priv = to_msm_priv(domain);
-        if (!priv) {
-                ret = -EINVAL;
-                goto fail;
-        }
        fl_table = priv->pgtable;
@@ -484,10 +486,7 @@ static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
        spin_lock_irqsave(&msm_iommu_lock, flags);
-        priv = domain->priv;
+        priv = to_msm_priv(domain);
-        if (!priv)
-                goto fail;
        fl_table = priv->pgtable;
@@ -566,7 +565,7 @@ static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
        spin_lock_irqsave(&msm_iommu_lock, flags);
-        priv = domain->priv;
+        priv = to_msm_priv(domain);
        if (list_empty(&priv->list_attached))
                goto fail;
@@ -674,8 +673,8 @@ fail:
 static const struct iommu_ops msm_iommu_ops = {
        .capable = msm_iommu_capable,
-        .domain_init = msm_iommu_domain_init,
+        .domain_alloc = msm_iommu_domain_alloc,
-        .domain_destroy = msm_iommu_domain_destroy,
+        .domain_free = msm_iommu_domain_free,
        .attach_dev = msm_iommu_attach_dev,
        .detach_dev = msm_iommu_detach_dev,
        .map = msm_iommu_map,
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index a4ba851825c2..a22c33d6a486 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -59,6 +59,7 @@ struct omap_iommu_domain {
        struct omap_iommu *iommu_dev;
        struct device *dev;
        spinlock_t lock;
+        struct iommu_domain domain;
 };
 #define MMU_LOCK_BASE_SHIFT     10
@@ -80,6 +81,15 @@ static struct platform_driver omap_iommu_driver;
 static struct kmem_cache *iopte_cachep;
 /**
+ * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
+ * @dom:        generic iommu domain handle
+ **/
+static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct omap_iommu_domain, domain);
+}
+/**
 * omap_iommu_save_ctx - Save registers for pm off-mode support
 * @dev:        client device
 **/
@@ -901,7 +911,7 @@ static irqreturn_t iommu_fault_handler(int irq, void *data)
        u32 *iopgd, *iopte;
        struct omap_iommu *obj = data;
        struct iommu_domain *domain = obj->domain;
-        struct omap_iommu_domain *omap_domain = domain->priv;
+        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        if (!omap_domain->iommu_dev)
                return IRQ_NONE;
@@ -1113,7 +1123,7 @@ static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
 static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
                         phys_addr_t pa, size_t bytes, int prot)
 {
-        struct omap_iommu_domain *omap_domain = domain->priv;
+        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        struct omap_iommu *oiommu = omap_domain->iommu_dev;
        struct device *dev = oiommu->dev;
        struct iotlb_entry e;
@@ -1140,7 +1150,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
 static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
                            size_t size)
 {
-        struct omap_iommu_domain *omap_domain = domain->priv;
+        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        struct omap_iommu *oiommu = omap_domain->iommu_dev;
        struct device *dev = oiommu->dev;
@@ -1152,7 +1162,7 @@ static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
 static int
 omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
 {
-        struct omap_iommu_domain *omap_domain = domain->priv;
+        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        struct omap_iommu *oiommu;
        struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
        int ret = 0;
@@ -1212,17 +1222,20 @@ static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
 static void omap_iommu_detach_dev(struct iommu_domain *domain,
                                 struct device *dev)
 {
-        struct omap_iommu_domain *omap_domain = domain->priv;
+        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        spin_lock(&omap_domain->lock);
        _omap_iommu_detach_dev(omap_domain, dev);
        spin_unlock(&omap_domain->lock);
 }
-static int omap_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
 {
        struct omap_iommu_domain *omap_domain;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
        omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
        if (!omap_domain) {
                pr_err("kzalloc failed\n");
@@ -1244,25 +1257,21 @@ static int omap_iommu_domain_init(struct iommu_domain *domain)
        clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
        spin_lock_init(&omap_domain->lock);
-        domain->priv = omap_domain;
+        omap_domain->domain.geometry.aperture_start = 0;
+        omap_domain->domain.geometry.aperture_end   = (1ULL << 32) - 1;
+        omap_domain->domain.geometry.force_aperture = true;
-        domain->geometry.aperture_start = 0;
+        return &omap_domain->domain;
-        domain->geometry.aperture_end   = (1ULL << 32) - 1;
-        domain->geometry.force_aperture = true;
-        return 0;
 fail_nomem:
        kfree(omap_domain);
 out:
-        return -ENOMEM;
+        return NULL;
 }
-static void omap_iommu_domain_destroy(struct iommu_domain *domain)
+static void omap_iommu_domain_free(struct iommu_domain *domain)
 {
-        struct omap_iommu_domain *omap_domain = domain->priv;
+        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
-        domain->priv = NULL;
        /*
         * An iommu device is still attached
@@ -1278,7 +1287,7 @@ static void omap_iommu_domain_destroy(struct iommu_domain *domain)
 static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
                                          dma_addr_t da)
 {
-        struct omap_iommu_domain *omap_domain = domain->priv;
+        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        struct omap_iommu *oiommu = omap_domain->iommu_dev;
        struct device *dev = oiommu->dev;
        u32 *pgd, *pte;
@@ -1358,8 +1367,8 @@ static void omap_iommu_remove_device(struct device *dev)
 }
 static const struct iommu_ops omap_iommu_ops = {
-        .domain_init    = omap_iommu_domain_init,
+        .domain_alloc   = omap_iommu_domain_alloc,
-        .domain_destroy = omap_iommu_domain_destroy,
+        .domain_free    = omap_iommu_domain_free,
        .attach_dev     = omap_iommu_attach_dev,
        .detach_dev     = omap_iommu_detach_dev,
        .map            = omap_iommu_map,
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index 9f74fddcd304..4015560bf486 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -80,6 +80,8 @@ struct rk_iommu_domain {
        u32 *dt; /* page directory table */
        spinlock_t iommus_lock; /* lock for iommus list */
        spinlock_t dt_lock; /* lock for modifying page directory table */
+        struct iommu_domain domain;
 };
 struct rk_iommu {
@@ -100,6 +102,11 @@ static inline void rk_table_flush(u32 *va, unsigned int count)
        outer_flush_range(pa_start, pa_end);
 }
+static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct rk_iommu_domain, domain);
+}
 /**
 * Inspired by _wait_for in intel_drv.h
 * This is NOT safe for use in interrupt context.
@@ -503,7 +510,7 @@ static irqreturn_t rk_iommu_irq(int irq, void *dev_id)
 static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain,
                                         dma_addr_t iova)
 {
-        struct rk_iommu_domain *rk_domain = domain->priv;
+        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        unsigned long flags;
        phys_addr_t pt_phys, phys = 0;
        u32 dte, pte;
@@ -639,7 +646,7 @@ unwind:
 static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
                        phys_addr_t paddr, size_t size, int prot)
 {
-        struct rk_iommu_domain *rk_domain = domain->priv;
+        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        unsigned long flags;
        dma_addr_t iova = (dma_addr_t)_iova;
        u32 *page_table, *pte_addr;
@@ -670,7 +677,7 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
 static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
                             size_t size)
 {
-        struct rk_iommu_domain *rk_domain = domain->priv;
+        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        unsigned long flags;
        dma_addr_t iova = (dma_addr_t)_iova;
        phys_addr_t pt_phys;
@@ -726,7 +733,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
                                  struct device *dev)
 {
        struct rk_iommu *iommu;
-        struct rk_iommu_domain *rk_domain = domain->priv;
+        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        unsigned long flags;
        int ret;
        phys_addr_t dte_addr;
@@ -778,7 +785,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
                                   struct device *dev)
 {
        struct rk_iommu *iommu;
-        struct rk_iommu_domain *rk_domain = domain->priv;
+        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        unsigned long flags;
        /* Allow 'virtual devices' (eg drm) to detach from domain */
@@ -804,13 +811,16 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
        dev_info(dev, "Detached from iommu domain\n");
 }
-static int rk_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
 {
        struct rk_iommu_domain *rk_domain;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
        rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL);
        if (!rk_domain)
-                return -ENOMEM;
+                return NULL;
        /*
         * rk32xx iommus use a 2 level pagetable.
@@ -827,17 +837,16 @@ static int rk_iommu_domain_init(struct iommu_domain *domain)
        spin_lock_init(&rk_domain->dt_lock);
        INIT_LIST_HEAD(&rk_domain->iommus);
-        domain->priv = rk_domain;
+        return &rk_domain->domain;
-        return 0;
 err_dt:
        kfree(rk_domain);
-        return -ENOMEM;
+        return NULL;
 }
-static void rk_iommu_domain_destroy(struct iommu_domain *domain)
+static void rk_iommu_domain_free(struct iommu_domain *domain)
 {
-        struct rk_iommu_domain *rk_domain = domain->priv;
+        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        int i;
        WARN_ON(!list_empty(&rk_domain->iommus));
@@ -852,8 +861,7 @@ static void rk_iommu_domain_destroy(struct iommu_domain *domain)
        }
        free_page((unsigned long)rk_domain->dt);
-        kfree(domain->priv);
+        kfree(rk_domain);
-        domain->priv = NULL;
 }
 static bool rk_iommu_is_dev_iommu_master(struct device *dev)
@@ -952,8 +960,8 @@ static void rk_iommu_remove_device(struct device *dev)
 }
 static const struct iommu_ops rk_iommu_ops = {
-        .domain_init = rk_iommu_domain_init,
+        .domain_alloc = rk_iommu_domain_alloc,
-        .domain_destroy = rk_iommu_domain_destroy,
+        .domain_free = rk_iommu_domain_free,
        .attach_dev = rk_iommu_attach_device,
        .detach_dev = rk_iommu_detach_device,
        .map = rk_iommu_map,
diff --git a/drivers/iommu/shmobile-iommu.c b/drivers/iommu/shmobile-iommu.c
index f1b00774e4de..a0287519a1d4 100644
--- a/drivers/iommu/shmobile-iommu.c
+++ b/drivers/iommu/shmobile-iommu.c
@@ -42,11 +42,17 @@ struct shmobile_iommu_domain {
        spinlock_t map_lock;
        spinlock_t attached_list_lock;
        struct list_head attached_list;
+        struct iommu_domain domain;
 };
 static struct shmobile_iommu_archdata *ipmmu_archdata;
 static struct kmem_cache *l1cache, *l2cache;
+static struct shmobile_iommu_domain *to_sh_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct shmobile_iommu_domain, domain);
+}
 static int pgtable_alloc(struct shmobile_iommu_domain_pgtable *pgtable,
                         struct kmem_cache *cache, size_t size)
 {
@@ -82,31 +88,33 @@ static void pgtable_write(struct shmobile_iommu_domain_pgtable *pgtable,
                                   sizeof(val) * count, DMA_TO_DEVICE);
 }
-static int shmobile_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *shmobile_iommu_domain_alloc(unsigned type)
 {
        struct shmobile_iommu_domain *sh_domain;
        int i, ret;
-        sh_domain = kmalloc(sizeof(*sh_domain), GFP_KERNEL);
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
+        sh_domain = kzalloc(sizeof(*sh_domain), GFP_KERNEL);
        if (!sh_domain)
-                return -ENOMEM;
+                return NULL;
        ret = pgtable_alloc(&sh_domain->l1, l1cache, L1_SIZE);
        if (ret < 0) {
                kfree(sh_domain);
-                return ret;
+                return NULL;
        }
        for (i = 0; i < L1_LEN; i++)
                sh_domain->l2[i].pgtable = NULL;
        spin_lock_init(&sh_domain->map_lock);
        spin_lock_init(&sh_domain->attached_list_lock);
        INIT_LIST_HEAD(&sh_domain->attached_list);
-        domain->priv = sh_domain;
+        return &sh_domain->domain;
-        return 0;
 }
-static void shmobile_iommu_domain_destroy(struct iommu_domain *domain)
+static void shmobile_iommu_domain_free(struct iommu_domain *domain)
 {
-        struct shmobile_iommu_domain *sh_domain = domain->priv;
+        struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
        int i;
        for (i = 0; i < L1_LEN; i++) {
@@ -115,14 +123,13 @@ static void shmobile_iommu_domain_destroy(struct iommu_domain *domain)
        }
        pgtable_free(&sh_domain->l1, l1cache, L1_SIZE);
        kfree(sh_domain);
-        domain->priv = NULL;
 }
 static int shmobile_iommu_attach_device(struct iommu_domain *domain,
                                        struct device *dev)
 {
        struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
-        struct shmobile_iommu_domain *sh_domain = domain->priv;
+        struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
        int ret = -EBUSY;
        if (!archdata)
@@ -151,7 +158,7 @@ static void shmobile_iommu_detach_device(struct iommu_domain *domain,
                                         struct device *dev)
 {
        struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
-        struct shmobile_iommu_domain *sh_domain = domain->priv;
+        struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
        if (!archdata)
                return;
@@ -214,7 +221,7 @@ static int shmobile_iommu_map(struct iommu_domain *domain, unsigned long iova,
                              phys_addr_t paddr, size_t size, int prot)
 {
        struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
-        struct shmobile_iommu_domain *sh_domain = domain->priv;
+        struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
        unsigned int l1index, l2index;
        int ret;
@@ -258,7 +265,7 @@ static size_t shmobile_iommu_unmap(struct iommu_domain *domain,
                                   unsigned long iova, size_t size)
 {
        struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
-        struct shmobile_iommu_domain *sh_domain = domain->priv;
+        struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
        unsigned int l1index, l2index;
        uint32_t l2entry = 0;
        size_t ret = 0;
@@ -298,7 +305,7 @@ done:
 static phys_addr_t shmobile_iommu_iova_to_phys(struct iommu_domain *domain,
                                               dma_addr_t iova)
 {
-        struct shmobile_iommu_domain *sh_domain = domain->priv;
+        struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
        uint32_t l1entry = 0, l2entry = 0;
        unsigned int l1index, l2index;
@@ -355,8 +362,8 @@ static int shmobile_iommu_add_device(struct device *dev)
 }
 static const struct iommu_ops shmobile_iommu_ops = {
-        .domain_init = shmobile_iommu_domain_init,
+        .domain_alloc = shmobile_iommu_domain_alloc,
-        .domain_destroy = shmobile_iommu_domain_destroy,
+        .domain_free = shmobile_iommu_domain_free,
        .attach_dev = shmobile_iommu_attach_device,
        .detach_dev = shmobile_iommu_detach_device,
        .map = shmobile_iommu_map,
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index c48da057dbb1..37e708fdbb5a 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -63,11 +63,21 @@ struct gart_device {
        struct device           *dev;
 };
+struct gart_domain {
+        struct iommu_domain domain;             /* generic domain handle */
+        struct gart_device *gart;               /* link to gart device   */
+};
 static struct gart_device *gart_handle; /* unique for a system */
 #define GART_PTE(_pfn)                                          \
        (GART_ENTRY_PHYS_ADDR_VALID | ((_pfn) << PAGE_SHIFT))
+static struct gart_domain *to_gart_domain(struct iommu_domain *dom)
+{
+        return container_of(dom, struct gart_domain, domain);
+}
 /*
 * Any interaction between any block on PPSB and a block on APB or AHB
 * must have these read-back to ensure the APB/AHB bus transaction is
@@ -156,20 +166,11 @@ static inline bool gart_iova_range_valid(struct gart_device *gart,
 static int gart_iommu_attach_dev(struct iommu_domain *domain,
                                 struct device *dev)
 {
-        struct gart_device *gart;
+        struct gart_domain *gart_domain = to_gart_domain(domain);
+        struct gart_device *gart = gart_domain->gart;
        struct gart_client *client, *c;
        int err = 0;
-        gart = gart_handle;
-        if (!gart)
-                return -EINVAL;
-        domain->priv = gart;
-        domain->geometry.aperture_start = gart->iovmm_base;
-        domain->geometry.aperture_end   = gart->iovmm_base +
-                                        gart->page_count * GART_PAGE_SIZE - 1;
-        domain->geometry.force_aperture = true;
        client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL);
        if (!client)
                return -ENOMEM;
@@ -198,7 +199,8 @@ fail:
 static void gart_iommu_detach_dev(struct iommu_domain *domain,
                                  struct device *dev)
 {
-        struct gart_device *gart = domain->priv;
+        struct gart_domain *gart_domain = to_gart_domain(domain);
+        struct gart_device *gart = gart_domain->gart;
        struct gart_client *c;
        spin_lock(&gart->client_lock);
@@ -216,33 +218,55 @@ out:
        spin_unlock(&gart->client_lock);
 }
-static int gart_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *gart_iommu_domain_alloc(unsigned type)
 {
-        return 0;
+        struct gart_domain *gart_domain;
-}
+        struct gart_device *gart;
-static void gart_iommu_domain_destroy(struct iommu_domain *domain)
+        if (type != IOMMU_DOMAIN_UNMANAGED)
-{
+                return NULL;
-        struct gart_device *gart = domain->priv;
+        gart = gart_handle;
        if (!gart)
-                return;
+                return NULL;
-        spin_lock(&gart->client_lock);
+        gart_domain = kzalloc(sizeof(*gart_domain), GFP_KERNEL);
-        if (!list_empty(&gart->client)) {
+        if (!gart_domain)
-                struct gart_client *c;
+                return NULL;
+        gart_domain->gart = gart;
+        gart_domain->domain.geometry.aperture_start = gart->iovmm_base;
+        gart_domain->domain.geometry.aperture_end = gart->iovmm_base +
+                                        gart->page_count * GART_PAGE_SIZE - 1;
+        gart_domain->domain.geometry.force_aperture = true;
+        return &gart_domain->domain;
+}
+static void gart_iommu_domain_free(struct iommu_domain *domain)
+{
+        struct gart_domain *gart_domain = to_gart_domain(domain);
+        struct gart_device *gart = gart_domain->gart;
+        if (gart) {
+                spin_lock(&gart->client_lock);
+                if (!list_empty(&gart->client)) {
+                        struct gart_client *c;
-                list_for_each_entry(c, &gart->client, list)
+                        list_for_each_entry(c, &gart->client, list)
-                        gart_iommu_detach_dev(domain, c->dev);
+                                gart_iommu_detach_dev(domain, c->dev);
+                }
+                spin_unlock(&gart->client_lock);
        }
-        spin_unlock(&gart->client_lock);
-        domain->priv = NULL;
+        kfree(gart_domain);
 }
 static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
                          phys_addr_t pa, size_t bytes, int prot)
 {
-        struct gart_device *gart = domain->priv;
+        struct gart_domain *gart_domain = to_gart_domain(domain);
+        struct gart_device *gart = gart_domain->gart;
        unsigned long flags;
        unsigned long pfn;
@@ -265,7 +289,8 @@ static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
 static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
                               size_t bytes)
 {
-        struct gart_device *gart = domain->priv;
+        struct gart_domain *gart_domain = to_gart_domain(domain);
+        struct gart_device *gart = gart_domain->gart;
        unsigned long flags;
        if (!gart_iova_range_valid(gart, iova, bytes))
@@ -281,7 +306,8 @@ static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
 static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
                                           dma_addr_t iova)
 {
-        struct gart_device *gart = domain->priv;
+        struct gart_domain *gart_domain = to_gart_domain(domain);
+        struct gart_device *gart = gart_domain->gart;
        unsigned long pte;
        phys_addr_t pa;
        unsigned long flags;
@@ -310,8 +336,8 @@ static bool gart_iommu_capable(enum iommu_cap cap)
 static const struct iommu_ops gart_iommu_ops = {
        .capable        = gart_iommu_capable,
-        .domain_init    = gart_iommu_domain_init,
+        .domain_alloc   = gart_iommu_domain_alloc,
-        .domain_destroy = gart_iommu_domain_destroy,
+        .domain_free    = gart_iommu_domain_free,
        .attach_dev     = gart_iommu_attach_dev,
        .detach_dev     = gart_iommu_detach_dev,
        .map            = gart_iommu_map,
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 6e134c7c227f..c845d99ecf6b 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -6,6 +6,7 @@
 * published by the Free Software Foundation.
 */
+#include <linux/bitops.h>
 #include <linux/err.h>
 #include <linux/iommu.h>
 #include <linux/kernel.h>
@@ -24,6 +25,8 @@ struct tegra_smmu {
        struct tegra_mc *mc;
        const struct tegra_smmu_soc *soc;
+        unsigned long pfn_mask;
        unsigned long *asids;
        struct mutex lock;
@@ -31,7 +34,7 @@ struct tegra_smmu {
 };
 struct tegra_smmu_as {
-        struct iommu_domain *domain;
+        struct iommu_domain domain;
        struct tegra_smmu *smmu;
        unsigned int use_count;
        struct page *count;
@@ -40,6 +43,11 @@ struct tegra_smmu_as {
        u32 attr;
 };
+static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
+{
+        return container_of(dom, struct tegra_smmu_as, domain);
+}
 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
                               unsigned long offset)
 {
@@ -105,8 +113,6 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
 #define SMMU_PDE_SHIFT 22
 #define SMMU_PTE_SHIFT 12
-#define SMMU_PFN_MASK 0x000fffff
 #define SMMU_PD_READABLE        (1 << 31)
 #define SMMU_PD_WRITABLE        (1 << 30)
 #define SMMU_PD_NONSECURE       (1 << 29)
@@ -224,30 +230,32 @@ static bool tegra_smmu_capable(enum iommu_cap cap)
        return false;
 }
-static int tegra_smmu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
 {
        struct tegra_smmu_as *as;
        unsigned int i;
        uint32_t *pd;
+        if (type != IOMMU_DOMAIN_UNMANAGED)
+                return NULL;
        as = kzalloc(sizeof(*as), GFP_KERNEL);
        if (!as)
-                return -ENOMEM;
+                return NULL;
        as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
-        as->domain = domain;
        as->pd = alloc_page(GFP_KERNEL | __GFP_DMA);
        if (!as->pd) {
                kfree(as);
-                return -ENOMEM;
+                return NULL;
        }
        as->count = alloc_page(GFP_KERNEL);
        if (!as->count) {
                __free_page(as->pd);
                kfree(as);
-                return -ENOMEM;
+                return NULL;
        }
        /* clear PDEs */
@@ -264,14 +272,17 @@ static int tegra_smmu_domain_init(struct iommu_domain *domain)
        for (i = 0; i < SMMU_NUM_PDE; i++)
                pd[i] = 0;
-        domain->priv = as;
+        /* setup aperture */
+        as->domain.geometry.aperture_start = 0;
+        as->domain.geometry.aperture_end = 0xffffffff;
+        as->domain.geometry.force_aperture = true;
-        return 0;
+        return &as->domain;
 }
-static void tegra_smmu_domain_destroy(struct iommu_domain *domain)
+static void tegra_smmu_domain_free(struct iommu_domain *domain)
 {
-        struct tegra_smmu_as *as = domain->priv;
+        struct tegra_smmu_as *as = to_smmu_as(domain);
        /* TODO: free page directory and page tables */
        ClearPageReserved(as->pd);
@@ -395,7 +406,7 @@ static int tegra_smmu_attach_dev(struct iommu_domain *domain,
                                 struct device *dev)
 {
        struct tegra_smmu *smmu = dev->archdata.iommu;
-        struct tegra_smmu_as *as = domain->priv;
+        struct tegra_smmu_as *as = to_smmu_as(domain);
        struct device_node *np = dev->of_node;
        struct of_phandle_args args;
        unsigned int index = 0;
@@ -428,7 +439,7 @@ static int tegra_smmu_attach_dev(struct iommu_domain *domain,
 static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
 {
-        struct tegra_smmu_as *as = domain->priv;
+        struct tegra_smmu_as *as = to_smmu_as(domain);
        struct device_node *np = dev->of_node;
        struct tegra_smmu *smmu = as->smmu;
        struct of_phandle_args args;
@@ -481,7 +492,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
                smmu_flush_tlb_section(smmu, as->id, iova);
                smmu_flush(smmu);
        } else {
-                page = pfn_to_page(pd[pde] & SMMU_PFN_MASK);
+                page = pfn_to_page(pd[pde] & smmu->pfn_mask);
                pt = page_address(page);
        }
@@ -503,7 +514,7 @@ static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova)
        u32 *pd = page_address(as->pd), *pt;
        struct page *page;
-        page = pfn_to_page(pd[pde] & SMMU_PFN_MASK);
+        page = pfn_to_page(pd[pde] & as->smmu->pfn_mask);
        pt = page_address(page);
        /*
@@ -524,7 +535,7 @@ static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova)
 static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
                          phys_addr_t paddr, size_t size, int prot)
 {
-        struct tegra_smmu_as *as = domain->priv;
+        struct tegra_smmu_as *as = to_smmu_as(domain);
        struct tegra_smmu *smmu = as->smmu;
        unsigned long offset;
        struct page *page;
@@ -548,7 +559,7 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
 static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
                               size_t size)
 {
-        struct tegra_smmu_as *as = domain->priv;
+        struct tegra_smmu_as *as = to_smmu_as(domain);
        struct tegra_smmu *smmu = as->smmu;
        unsigned long offset;
        struct page *page;
@@ -572,13 +583,13 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
 static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
                                           dma_addr_t iova)
 {
-        struct tegra_smmu_as *as = domain->priv;
+        struct tegra_smmu_as *as = to_smmu_as(domain);
        struct page *page;
        unsigned long pfn;
        u32 *pte;
        pte = as_get_pte(as, iova, &page);
-        pfn = *pte & SMMU_PFN_MASK;
+        pfn = *pte & as->smmu->pfn_mask;
        return PFN_PHYS(pfn);
 }
@@ -633,8 +644,8 @@ static void tegra_smmu_remove_device(struct device *dev)
 static const struct iommu_ops tegra_smmu_ops = {
        .capable = tegra_smmu_capable,
-        .domain_init = tegra_smmu_domain_init,
+        .domain_alloc = tegra_smmu_domain_alloc,
-        .domain_destroy = tegra_smmu_domain_destroy,
+        .domain_free = tegra_smmu_domain_free,
        .attach_dev = tegra_smmu_attach_dev,
        .detach_dev = tegra_smmu_detach_dev,
        .add_device = tegra_smmu_add_device,
@@ -702,6 +713,10 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
        smmu->dev = dev;
        smmu->mc = mc;
+        smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;
+        dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
+                mc->soc->num_address_bits, smmu->pfn_mask);
        value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
        if (soc->supports_request_limit)
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 4f2fb62e6f37..49875adb6b44 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -567,7 +567,7 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
         */
        smp_wmb();
-        for_each_cpu_mask(cpu, *mask) {
+        for_each_cpu(cpu, mask) {
                u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
                u16 tlist;
diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index bc48b7dc89ec..57f09cb54464 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -389,19 +389,19 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
        int             i;
        cpumask_and(&tmp, cpumask, cpu_online_mask);
-        if (cpus_empty(tmp))
+        if (cpumask_empty(&tmp))
                return -EINVAL;
        /* Assumption : cpumask refers to a single CPU */
        spin_lock_irqsave(&gic_lock, flags);
        /* Re-route this IRQ */
-        gic_map_to_vpe(irq, first_cpu(tmp));
+        gic_map_to_vpe(irq, cpumask_first(&tmp));
        /* Update the pcpu_masks */
        for (i = 0; i < NR_CPUS; i++)
                clear_bit(irq, pcpu_masks[i].pcpu_mask);
-        set_bit(irq, pcpu_masks[first_cpu(tmp)].pcpu_mask);
+        set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
        cpumask_copy(d->affinity, cpumask);
        spin_unlock_irqrestore(&gic_lock, flags);
diff --git a/drivers/net/ethernet/tile/tilegx.c b/drivers/net/ethernet/tile/tilegx.c
index a789a2054388..a3f7610002aa 100644
--- a/drivers/net/ethernet/tile/tilegx.c
+++ b/drivers/net/ethernet/tile/tilegx.c
@@ -1123,7 +1123,7 @@ static int alloc_percpu_mpipe_resources(struct net_device *dev,
                        addr + i * sizeof(struct tile_net_comps);
        /* If this is a network cpu, create an iqueue. */
-        if (cpu_isset(cpu, network_cpus_map)) {
+        if (cpumask_test_cpu(cpu, &network_cpus_map)) {
                order = get_order(NOTIF_RING_SIZE);
                page = homecache_alloc_pages(GFP_KERNEL, order, cpu);
                if (page == NULL) {
@@ -1299,7 +1299,7 @@ static int tile_net_init_mpipe(struct net_device *dev)
        int first_ring, ring;
        int instance = mpipe_instance(dev);
        struct mpipe_data *md = &mpipe_data[instance];
-        int network_cpus_count = cpus_weight(network_cpus_map);
+        int network_cpus_count = cpumask_weight(&network_cpus_map);
        if (!hash_default) {
                netdev_err(dev, "Networking requires hash_default!\n");
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index 26a51dc4278d..57fd66357b95 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -579,7 +579,8 @@ void dasd_kick_device(struct dasd_device *device)
 {
        dasd_get_device(device);
        /* queue call to dasd_kick_device to the kernel event daemon. */
-        schedule_work(&device->kick_work);
+        if (!schedule_work(&device->kick_work))
+                dasd_put_device(device);
 }
 EXPORT_SYMBOL(dasd_kick_device);
@@ -599,7 +600,8 @@ void dasd_reload_device(struct dasd_device *device)
 {
        dasd_get_device(device);
        /* queue call to dasd_reload_device to the kernel event daemon. */
-        schedule_work(&device->reload_device);
+        if (!schedule_work(&device->reload_device))
+                dasd_put_device(device);
 }
 EXPORT_SYMBOL(dasd_reload_device);
@@ -619,7 +621,8 @@ void dasd_restore_device(struct dasd_device *device)
 {
        dasd_get_device(device);
        /* queue call to dasd_restore_device to the kernel event daemon. */
-        schedule_work(&device->restore_device);
+        if (!schedule_work(&device->restore_device))
+                dasd_put_device(device);
 }
 /*
@@ -2163,18 +2166,22 @@ static int _dasd_sleep_on(struct dasd_ccw_req *maincqr, int interruptible)
                        cqr->intrc = -ENOLINK;
                        continue;
                }
-                /* Don't try to start requests if device is stopped */
+                /*
-                if (interruptible) {
+                 * Don't try to start requests if device is stopped
-                        rc = wait_event_interruptible(
+                 * except path verification requests
-                                generic_waitq, !(device->stopped));
+                 */
-                        if (rc == -ERESTARTSYS) {
+                if (!test_bit(DASD_CQR_VERIFY_PATH, &cqr->flags)) {
-                                cqr->status = DASD_CQR_FAILED;
+                        if (interruptible) {
-                                maincqr->intrc = rc;
+                                rc = wait_event_interruptible(
-                                continue;
+                                        generic_waitq, !(device->stopped));
-                        }
+                                if (rc == -ERESTARTSYS) {
-                } else
+                                        cqr->status = DASD_CQR_FAILED;
-                        wait_event(generic_waitq, !(device->stopped));
+                                        maincqr->intrc = rc;
+                                        continue;
+                                }
+                        } else
+                                wait_event(generic_waitq, !(device->stopped));
+                }
                if (!cqr->callback)
                        cqr->callback = dasd_wakeup_cb;
@@ -2524,6 +2531,11 @@ static void __dasd_process_request_queue(struct dasd_block *block)
                        __blk_end_request_all(req, -EIO);
                return;
        }
+        /* if device ist stopped do not fetch new requests */
+        if (basedev->stopped)
+                return;
        /* Now we try to fetch requests from the request queue */
        while ((req = blk_peek_request(queue))) {
                if (basedev->features & DASD_FEATURE_READONLY &&
diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
index 49b48a887c66..6215f6455eb8 100644
--- a/drivers/s390/block/dasd_eckd.c
+++ b/drivers/s390/block/dasd_eckd.c
@@ -1628,7 +1628,8 @@ static void dasd_eckd_kick_validate_server(struct dasd_device *device)
                return;
        }
        /* queue call to do_validate_server to the kernel event daemon. */
-        schedule_work(&device->kick_validate);
+        if (!schedule_work(&device->kick_validate))
+                dasd_put_device(device);
 }
 static u32 get_fcx_max_data(struct dasd_device *device)
diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c
index 6e14999f9e8f..7be782116dab 100644
--- a/drivers/s390/char/sclp_cmd.c
+++ b/drivers/s390/char/sclp_cmd.c
@@ -315,10 +315,29 @@ static int sclp_mem_change_state(unsigned long start, unsigned long size,
                        rc |= sclp_assign_storage(incr->rn);
                else
                        sclp_unassign_storage(incr->rn);
+                if (rc == 0)
+                        incr->standby = online ? 0 : 1;
        }
        return rc ? -EIO : 0;
 }
+static bool contains_standby_increment(unsigned long start, unsigned long end)
+{
+        struct memory_increment *incr;
+        unsigned long istart;
+        list_for_each_entry(incr, &sclp_mem_list, list) {
+                istart = rn2addr(incr->rn);
+                if (end - 1 < istart)
+                        continue;
+                if (start > istart + sclp_rzm - 1)
+                        continue;
+                if (incr->standby)
+                        return true;
+        }
+        return false;
+}
 static int sclp_mem_notifier(struct notifier_block *nb,
                             unsigned long action, void *data)
 {
@@ -334,8 +353,16 @@ static int sclp_mem_notifier(struct notifier_block *nb,
        for_each_clear_bit(id, sclp_storage_ids, sclp_max_storage_id + 1)
                sclp_attach_storage(id);
        switch (action) {
-        case MEM_ONLINE:
        case MEM_GOING_OFFLINE:
+                /*
+                 * We do not allow to set memory blocks offline that contain
+                 * standby memory. This is done to simplify the "memory online"
+                 * case.
+                 */
+                if (contains_standby_increment(start, start + size))
+                        rc = -EPERM;
+                break;
+        case MEM_ONLINE:
        case MEM_CANCEL_OFFLINE:
                break;
        case MEM_GOING_ONLINE:
@@ -361,6 +388,21 @@ static struct notifier_block sclp_mem_nb = {
        .notifier_call = sclp_mem_notifier,
 };
+static void __init align_to_block_size(unsigned long long *start,
+                                       unsigned long long *size)
+{
+        unsigned long long start_align, size_align, alignment;
+        alignment = memory_block_size_bytes();
+        start_align = roundup(*start, alignment);
+        size_align = rounddown(*start + *size, alignment) - start_align;
+        pr_info("Standby memory at 0x%llx (%lluM of %lluM usable)\n",
+                *start, size_align >> 20, *size >> 20);
+        *start = start_align;
+        *size = size_align;
+}
 static void __init add_memory_merged(u16 rn)
 {
        static u16 first_rn, num;
@@ -382,7 +424,9 @@ static void __init add_memory_merged(u16 rn)
                goto skip_add;
        if (memory_end_set && (start + size > memory_end))
                size = memory_end - start;
-        add_memory(0, start, size);
+        align_to_block_size(&start, &size);
+        if (size)
+                add_memory(0, start, size);
 skip_add:
        first_rn = rn;
        num = 1;
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index a1cfbd3dda47..8eab107b53fb 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -6632,14 +6632,12 @@ static void fail_all_outstanding_cmds(struct ctlr_info *h)
 static void set_lockup_detected_for_all_cpus(struct ctlr_info *h, u32 value)
 {
-        int i, cpu;
+        int cpu;
-        cpu = cpumask_first(cpu_online_mask);
+        for_each_online_cpu(cpu) {
-        for (i = 0; i < num_online_cpus(); i++) {
                u32 *lockup_detected;
                lockup_detected = per_cpu_ptr(h->lockup_detected, cpu);
                *lockup_detected = value;
-                cpu = cpumask_next(cpu, cpu_online_mask);
        }
        wmb(); /* be sure the per-cpu variables are out to memory */
 }
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index 086549a665e2..27e285b92b5f 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -11,6 +11,7 @@
 #include <linux/bitmap.h>
 #include <linux/bug.h>
+/* Don't assign or return these: may not be this big! */
 typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
 /**
@@ -289,11 +290,11 @@ static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
 * @cpumask: the cpumask pointer
 *
 * Returns 1 if @cpu is set in @cpumask, else returns 0
- *
- * No static inline type checking - see Subtlety (1) above.
 */
-#define cpumask_test_cpu(cpu, cpumask) \
+static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
-        test_bit(cpumask_check(cpu), cpumask_bits((cpumask)))
+{
+        return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
+}
 /**
 * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
@@ -609,9 +610,7 @@ static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
 */
 static inline size_t cpumask_size(void)
 {
-        /* FIXME: Once all cpumask assignments are eliminated, this
+        return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
-         * can be nr_cpumask_bits */
-        return BITS_TO_LONGS(NR_CPUS) * sizeof(long);
 }
 /*
@@ -768,7 +767,7 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
 #if NR_CPUS <= BITS_PER_LONG
 #define CPU_BITS_ALL                                            \
 {                                                               \
-        [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
+        [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)     \
 }
 #else /* NR_CPUS > BITS_PER_LONG */
@@ -776,7 +775,7 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
 #define CPU_BITS_ALL                                            \
 {                                                               \
        [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,                \
-        [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD         \
+        [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)     \
 }
 #endif /* NR_CPUS > BITS_PER_LONG */
@@ -797,32 +796,18 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
                                      nr_cpu_ids);
 }
-/*
- *
- * From here down, all obsolete.  Use cpumask_ variants!
- *
- */
-#ifndef CONFIG_DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
-#define cpumask_of_cpu(cpu) (*get_cpu_mask(cpu))
-#define CPU_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(NR_CPUS)
 #if NR_CPUS <= BITS_PER_LONG
 #define CPU_MASK_ALL                                                    \
 (cpumask_t) { {                                                         \
-        [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD                 \
+        [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)     \
 } }
 #else
 #define CPU_MASK_ALL                                                    \
 (cpumask_t) { {                                                         \
        [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,                        \
-        [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD                 \
+        [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS)     \
 } }
+#endif /* NR_CPUS > BITS_PER_LONG */
-#endif
 #define CPU_MASK_NONE                                                   \
 (cpumask_t) { {                                                         \
@@ -834,143 +819,4 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
        [0] =  1UL                                                      \
 } }
-#if NR_CPUS == 1
-#define first_cpu(src)          ({ (void)(src); 0; })
-#define next_cpu(n, src)        ({ (void)(src); 1; })
-#define any_online_cpu(mask)    0
-#define for_each_cpu_mask(cpu, mask)    \
-        for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
-#else /* NR_CPUS > 1 */
-int __first_cpu(const cpumask_t *srcp);
-int __next_cpu(int n, const cpumask_t *srcp);
-#define first_cpu(src)          __first_cpu(&(src))
-#define next_cpu(n, src)        __next_cpu((n), &(src))
-#define any_online_cpu(mask) cpumask_any_and(&mask, cpu_online_mask)
-#define for_each_cpu_mask(cpu, mask)                    \
-        for ((cpu) = -1;                                \
-                (cpu) = next_cpu((cpu), (mask)),        \
-                (cpu) < NR_CPUS; )
-#endif /* SMP */
-#if NR_CPUS <= 64
-#define for_each_cpu_mask_nr(cpu, mask) for_each_cpu_mask(cpu, mask)
-#else /* NR_CPUS > 64 */
-int __next_cpu_nr(int n, const cpumask_t *srcp);
-#define for_each_cpu_mask_nr(cpu, mask)                 \
-        for ((cpu) = -1;                                \
-                (cpu) = __next_cpu_nr((cpu), &(mask)),  \
-                (cpu) < nr_cpu_ids; )
-#endif /* NR_CPUS > 64 */
-#define cpus_addr(src) ((src).bits)
-#define cpu_set(cpu, dst) __cpu_set((cpu), &(dst))
-static inline void __cpu_set(int cpu, volatile cpumask_t *dstp)
-{
-        set_bit(cpu, dstp->bits);
-}
-#define cpu_clear(cpu, dst) __cpu_clear((cpu), &(dst))
-static inline void __cpu_clear(int cpu, volatile cpumask_t *dstp)
-{
-        clear_bit(cpu, dstp->bits);
-}
-#define cpus_setall(dst) __cpus_setall(&(dst), NR_CPUS)
-static inline void __cpus_setall(cpumask_t *dstp, unsigned int nbits)
-{
-        bitmap_fill(dstp->bits, nbits);
-}
-#define cpus_clear(dst) __cpus_clear(&(dst), NR_CPUS)
-static inline void __cpus_clear(cpumask_t *dstp, unsigned int nbits)
-{
-        bitmap_zero(dstp->bits, nbits);
-}
-/* No static inline type checking - see Subtlety (1) above. */
-#define cpu_isset(cpu, cpumask) test_bit((cpu), (cpumask).bits)
-#define cpu_test_and_set(cpu, cpumask) __cpu_test_and_set((cpu), &(cpumask))
-static inline int __cpu_test_and_set(int cpu, cpumask_t *addr)
-{
-        return test_and_set_bit(cpu, addr->bits);
-}
-#define cpus_and(dst, src1, src2) __cpus_and(&(dst), &(src1), &(src2), NR_CPUS)
-static inline int __cpus_and(cpumask_t *dstp, const cpumask_t *src1p,
-                                        const cpumask_t *src2p, unsigned int nbits)
-{
-        return bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
-}
-#define cpus_or(dst, src1, src2) __cpus_or(&(dst), &(src1), &(src2), NR_CPUS)
-static inline void __cpus_or(cpumask_t *dstp, const cpumask_t *src1p,
-                                        const cpumask_t *src2p, unsigned int nbits)
-{
-        bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
-}
-#define cpus_xor(dst, src1, src2) __cpus_xor(&(dst), &(src1), &(src2), NR_CPUS)
-static inline void __cpus_xor(cpumask_t *dstp, const cpumask_t *src1p,
-                                        const cpumask_t *src2p, unsigned int nbits)
-{
-        bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
-}
-#define cpus_andnot(dst, src1, src2) \
-                                __cpus_andnot(&(dst), &(src1), &(src2), NR_CPUS)
-static inline int __cpus_andnot(cpumask_t *dstp, const cpumask_t *src1p,
-                                        const cpumask_t *src2p, unsigned int nbits)
-{
-        return bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
-}
-#define cpus_equal(src1, src2) __cpus_equal(&(src1), &(src2), NR_CPUS)
-static inline int __cpus_equal(const cpumask_t *src1p,
-                                        const cpumask_t *src2p, unsigned int nbits)
-{
-        return bitmap_equal(src1p->bits, src2p->bits, nbits);
-}
-#define cpus_intersects(src1, src2) __cpus_intersects(&(src1), &(src2), NR_CPUS)
-static inline int __cpus_intersects(const cpumask_t *src1p,
-                                        const cpumask_t *src2p, unsigned int nbits)
-{
-        return bitmap_intersects(src1p->bits, src2p->bits, nbits);
-}
-#define cpus_subset(src1, src2) __cpus_subset(&(src1), &(src2), NR_CPUS)
-static inline int __cpus_subset(const cpumask_t *src1p,
-                                        const cpumask_t *src2p, unsigned int nbits)
-{
-        return bitmap_subset(src1p->bits, src2p->bits, nbits);
-}
-#define cpus_empty(src) __cpus_empty(&(src), NR_CPUS)
-static inline int __cpus_empty(const cpumask_t *srcp, unsigned int nbits)
-{
-        return bitmap_empty(srcp->bits, nbits);
-}
-#define cpus_weight(cpumask) __cpus_weight(&(cpumask), NR_CPUS)
-static inline int __cpus_weight(const cpumask_t *srcp, unsigned int nbits)
-{
-        return bitmap_weight(srcp->bits, nbits);
-}
-#define cpus_shift_left(dst, src, n) \
-                        __cpus_shift_left(&(dst), &(src), (n), NR_CPUS)
-static inline void __cpus_shift_left(cpumask_t *dstp,
-                                        const cpumask_t *srcp, int n, int nbits)
-{
-        bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
-}
-#endif /* !CONFIG_DISABLE_OBSOLETE_CPUMASK_FUNCTIONS */
 #endif /* __LINUX_CPUMASK_H */
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 38daa453f2e5..0546b8710ce3 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -51,9 +51,33 @@ struct iommu_domain_geometry {
        bool force_aperture;       /* DMA only allowed in mappable range? */
 };
+/* Domain feature flags */
+#define __IOMMU_DOMAIN_PAGING   (1U << 0)  /* Support for iommu_map/unmap */
+#define __IOMMU_DOMAIN_DMA_API  (1U << 1)  /* Domain for use in DMA-API
+                                              implementation              */
+#define __IOMMU_DOMAIN_PT       (1U << 2)  /* Domain is identity mapped   */
+/*
+ * This are the possible domain-types
+ *
+ *      IOMMU_DOMAIN_BLOCKED    - All DMA is blocked, can be used to isolate
+ *                                devices
+ *      IOMMU_DOMAIN_IDENTITY   - DMA addresses are system physical addresses
+ *      IOMMU_DOMAIN_UNMANAGED  - DMA mappings managed by IOMMU-API user, used
+ *                                for VMs
+ *      IOMMU_DOMAIN_DMA        - Internally used for DMA-API implementations.
+ *                                This flag allows IOMMU drivers to implement
+ *                                certain optimizations for these domains
+ */
+#define IOMMU_DOMAIN_BLOCKED    (0U)
+#define IOMMU_DOMAIN_IDENTITY   (__IOMMU_DOMAIN_PT)
+#define IOMMU_DOMAIN_UNMANAGED  (__IOMMU_DOMAIN_PAGING)
+#define IOMMU_DOMAIN_DMA        (__IOMMU_DOMAIN_PAGING |        \
+                                 __IOMMU_DOMAIN_DMA_API)
 struct iommu_domain {
+        unsigned type;
        const struct iommu_ops *ops;
-        void *priv;
        iommu_fault_handler_t handler;
        void *handler_token;
        struct iommu_domain_geometry geometry;
@@ -113,8 +137,11 @@ enum iommu_attr {
 */
 struct iommu_ops {
        bool (*capable)(enum iommu_cap);
-        int (*domain_init)(struct iommu_domain *domain);
-        void (*domain_destroy)(struct iommu_domain *domain);
+        /* Domain allocation and freeing by the iommu driver */
+        struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type);
+        void (*domain_free)(struct iommu_domain *);
        int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
        void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
        int (*map)(struct iommu_domain *domain, unsigned long iova,
diff --git a/include/linux/smp.h b/include/linux/smp.h
index be91db2a7017..c4414074bd88 100644
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -18,7 +18,7 @@ struct call_single_data {
        struct llist_node llist;
        smp_call_func_t func;
        void *info;
-        u16 flags;
+        unsigned int flags;
 };
 /* total number of cpus in this system (may exceed NR_CPUS) */
diff --git a/lib/Kconfig b/lib/Kconfig
index f5440221d929..601965a948e8 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -396,10 +396,6 @@ config CPUMASK_OFFSTACK
          them on the stack.  This is a bit more expensive, but avoids
          stack overflow.
-config DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
-       bool "Disable obsolete cpumask functions" if DEBUG_PER_CPU_MAPS
-       depends on BROKEN
 config CPU_RMAP
        bool
        depends on SMP
diff --git a/lib/cpumask.c b/lib/cpumask.c
index 5ab1553fd076..830dd5dec40f 100644
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -5,27 +5,6 @@
 #include <linux/export.h>
 #include <linux/bootmem.h>
-int __first_cpu(const cpumask_t *srcp)
-{
-        return min_t(int, NR_CPUS, find_first_bit(srcp->bits, NR_CPUS));
-}
-EXPORT_SYMBOL(__first_cpu);
-int __next_cpu(int n, const cpumask_t *srcp)
-{
-        return min_t(int, NR_CPUS, find_next_bit(srcp->bits, NR_CPUS, n+1));
-}
-EXPORT_SYMBOL(__next_cpu);
-#if NR_CPUS > 64
-int __next_cpu_nr(int n, const cpumask_t *srcp)
-{
-        return min_t(int, nr_cpu_ids,
-                                find_next_bit(srcp->bits, nr_cpu_ids, n+1));
-}
-EXPORT_SYMBOL(__next_cpu_nr);
-#endif
 /**
 * cpumask_next_and - get the next cpu in *src1p & *src2p
 * @n: the cpu prior to the place to search (ie. return will be > @n)
@@ -90,13 +69,6 @@ bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
                dump_stack();
        }
 #endif
-        /* FIXME: Bandaid to save us from old primitives which go to NR_CPUS. */
-        if (*mask) {
-                unsigned char *ptr = (unsigned char *)cpumask_bits(*mask);
-                unsigned int tail;
-                tail = BITS_TO_LONGS(NR_CPUS - nr_cpumask_bits) * sizeof(long);
-                memset(ptr + cpumask_size() - tail, 0, tail);
-        }
        return *mask != NULL;
 }