ARM: 7259/3: net: JIT compiler for packet filters

Based of Matt Evans's PPC64 implementation.

The compiler generates ARM instructions but interworking is
supported for Thumb2 kernels.

Supports both little and big endian. Unaligned loads are emitted
for ARMv6+. Not all the BPF opcodes that deal with ancillary data
are supported. The scratch memory of the filter lives on the stack.
Hardware integer division is used if it is available.

Enabled in the same way as for x86-64 and PPC64:

	echo 1 > /proc/sys/net/core/bpf_jit_enable

A value greater than 1 enables opcode output.

Signed-off-by: Mircea Gherzan <mgherzan@gmail.com>
Acked-by: David S. Miller <davem@davemloft.net>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

5 files changed, 1110 insertions, 0 deletions

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index ee6c7154c9a1..746ab6d6a205 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -34,6 +34,7 @@ config ARM
         select GENERIC_IRQ_SHOW
         select CPU_PM if (SUSPEND || CPU_IDLE)
         select GENERIC_PCI_IOMAP
+        select HAVE_BPF_JIT
         help
           The ARM series is a line of low-power-consumption RISC chip designs
           licensed by ARM Ltd and targeted at embedded applications and
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index 40319d91bb7f..93d63be8fa59 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -253,6 +253,7 @@ core-$(CONFIG_VFP)		+= arch/arm/vfp/
 
 # If we have a machine-specific directory, then include it in the build.
 core-y                          += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/
+core-y                          += arch/arm/net/
 core-y                          += $(machdirs) $(platdirs)
 
 drivers-$(CONFIG_OPROFILE)      += arch/arm/oprofile/
diff --git a/arch/arm/net/Makefile b/arch/arm/net/Makefile
new file mode 100644
index 000000000000..c2c10841b6be
--- /dev/null
+++ b/arch/arm/net/Makefile
@@ -0,0 +1,3 @@
+# ARM-specific networking code
+
+obj-$(CONFIG_BPF_JIT) += bpf_jit_32.o
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
new file mode 100644
index 000000000000..62135849f48b
--- /dev/null
+++ b/arch/arm/net/bpf_jit_32.c
@@ -0,0 +1,915 @@
+/*
+ * Just-In-Time compiler for BPF filters on 32bit ARM
+ *
+ * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/bitops.h>
+#include <linux/compiler.h>
+#include <linux/errno.h>
+#include <linux/filter.h>
+#include <linux/moduleloader.h>
+#include <linux/netdevice.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <asm/cacheflush.h>
+#include <asm/hwcap.h>
+
+#include "bpf_jit_32.h"
+
+/*
+ * ABI:
+ *
+ * r0   scratch register
+ * r4   BPF register A
+ * r5   BPF register X
+ * r6   pointer to the skb
+ * r7   skb->data
+ * r8   skb_headlen(skb)
+ */
+
+#define r_scratch       ARM_R0
+/* r1-r3 are (also) used for the unaligned loads on the non-ARMv7 slowpath */
+#define r_off           ARM_R1
+#define r_A             ARM_R4
+#define r_X             ARM_R5
+#define r_skb           ARM_R6
+#define r_skb_data      ARM_R7
+#define r_skb_hl        ARM_R8
+
+#define SCRATCH_SP_OFFSET       0
+#define SCRATCH_OFF(k)          (SCRATCH_SP_OFFSET + (k))
+
+#define SEEN_MEM                ((1 << BPF_MEMWORDS) - 1)
+#define SEEN_MEM_WORD(k)        (1 << (k))
+#define SEEN_X                  (1 << BPF_MEMWORDS)
+#define SEEN_CALL               (1 << (BPF_MEMWORDS + 1))
+#define SEEN_SKB                (1 << (BPF_MEMWORDS + 2))
+#define SEEN_DATA               (1 << (BPF_MEMWORDS + 3))
+
+#define FLAG_NEED_X_RESET       (1 << 0)
+
+struct jit_ctx {
+        const struct sk_filter *skf;
+        unsigned idx;
+        unsigned prologue_bytes;
+        int ret0_fp_idx;
+        u32 seen;
+        u32 flags;
+        u32 *offsets;
+        u32 *target;
+#if __LINUX_ARM_ARCH__ < 7
+        u16 epilogue_bytes;
+        u16 imm_count;
+        u32 *imms;
+#endif
+};
+
+int bpf_jit_enable __read_mostly;
+
+static u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset)
+{
+        u8 ret;
+        int err;
+
+        err = skb_copy_bits(skb, offset, &ret, 1);
+
+        return (u64)err << 32 | ret;
+}
+
+static u64 jit_get_skb_h(struct sk_buff *skb, unsigned offset)
+{
+        u16 ret;
+        int err;
+
+        err = skb_copy_bits(skb, offset, &ret, 2);
+
+        return (u64)err << 32 | ntohs(ret);
+}
+
+static u64 jit_get_skb_w(struct sk_buff *skb, unsigned offset)
+{
+        u32 ret;
+        int err;
+
+        err = skb_copy_bits(skb, offset, &ret, 4);
+
+        return (u64)err << 32 | ntohl(ret);
+}
+
+/*
+ * Wrapper that handles both OABI and EABI and assures Thumb2 interworking
+ * (where the assembly routines like __aeabi_uidiv could cause problems).
+ */
+static u32 jit_udiv(u32 dividend, u32 divisor)
+{
+        return dividend / divisor;
+}
+
+static inline void _emit(int cond, u32 inst, struct jit_ctx *ctx)
+{
+        if (ctx->target != NULL)
+                ctx->target[ctx->idx] = inst | (cond << 28);
+
+        ctx->idx++;
+}
+
+/*
+ * Emit an instruction that will be executed unconditionally.
+ */
+static inline void emit(u32 inst, struct jit_ctx *ctx)
+{
+        _emit(ARM_COND_AL, inst, ctx);
+}
+
+static u16 saved_regs(struct jit_ctx *ctx)
+{
+        u16 ret = 0;
+
+        if ((ctx->skf->len > 1) ||
+            (ctx->skf->insns[0].code == BPF_S_RET_A))
+                ret |= 1 << r_A;
+
+#ifdef CONFIG_FRAME_POINTER
+        ret |= (1 << ARM_FP) | (1 << ARM_IP) | (1 << ARM_LR) | (1 << ARM_PC);
+#else
+        if (ctx->seen & SEEN_CALL)
+                ret |= 1 << ARM_LR;
+#endif
+        if (ctx->seen & (SEEN_DATA | SEEN_SKB))
+                ret |= 1 << r_skb;
+        if (ctx->seen & SEEN_DATA)
+                ret |= (1 << r_skb_data) | (1 << r_skb_hl);
+        if (ctx->seen & SEEN_X)
+                ret |= 1 << r_X;
+
+        return ret;
+}
+
+static inline int mem_words_used(struct jit_ctx *ctx)
+{
+        /* yes, we do waste some stack space IF there are "holes" in the set" */
+        return fls(ctx->seen & SEEN_MEM);
+}
+
+static inline bool is_load_to_a(u16 inst)
+{
+        switch (inst) {
+        case BPF_S_LD_W_LEN:
+        case BPF_S_LD_W_ABS:
+        case BPF_S_LD_H_ABS:
+        case BPF_S_LD_B_ABS: