From 2e23e3f0cc7c3249b510e94b5b3ec92577b67e81 Mon Sep 17 00:00:00 2001 From: Namhoon Kim Date: Wed, 23 Mar 2016 07:12:48 -0400 Subject: initial --- arch/arm/boot/compressed/Makefile | 1 + arch/arm/include/asm/unistd.h | 2 +- arch/arm/kernel/calls.S | 1 + arch/arm/mm/cache-l2x0.c | 10 +- arch/x86/syscalls/syscall_32.tbl | 1 + arch/x86/syscalls/syscall_64.tbl | 1 + include/litmus/cache_proc.h | 11 + include/litmus/unistd_32.h | 3 +- include/litmus/unistd_64.h | 4 +- litmus/Makefile | 3 +- litmus/cache_proc.c | 965 ++++++++++++++++++++++++++++++++++++++ litmus/litmus.c | 39 ++ 12 files changed, 1036 insertions(+), 5 deletions(-) create mode 100644 include/litmus/cache_proc.h create mode 100644 litmus/cache_proc.c diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 6e1fb2b2ecc7..e2284fef1ce3 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -107,6 +107,7 @@ ifeq ($(CONFIG_FUNCTION_TRACER),y) ORIG_CFLAGS := $(KBUILD_CFLAGS) KBUILD_CFLAGS = $(subst -pg, , $(ORIG_CFLAGS)) endif +KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING ccflags-y := -fpic -mno-single-pic-base -fno-builtin -I$(obj) asflags-y := -DZIMAGE diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h index 7197bbe4dda1..3b7d36b921d3 100644 --- a/arch/arm/include/asm/unistd.h +++ b/arch/arm/include/asm/unistd.h @@ -19,7 +19,7 @@ * This may need to be greater than __NR_last_syscall+1 in order to * account for the padding in the syscall table */ -#define __NR_syscalls (388 + NR_litmus_syscalls + 3) +#define __NR_syscalls (388 + NR_litmus_syscalls + 2) /* diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S index 3e002969469a..a272b84a2fe7 100644 --- a/arch/arm/kernel/calls.S +++ b/arch/arm/kernel/calls.S @@ -410,6 +410,7 @@ CALL(sys_release_ts) CALL(sys_null_call) /* 400 */ CALL(sys_get_current_budget) + CALL(sys_test_call) #ifndef syscalls_counted diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index e309c8f35af5..71c969a1d790 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -33,6 +33,8 @@ #include "cache-tauros3.h" #include "cache-aurora-l2.h" +#include + struct l2c_init_data { const char *type; unsigned way_size_0; @@ -726,7 +728,6 @@ static void __init l2c310_fixup(void __iomem *base, u32 cache_id, if (n) { unsigned i; - pr_info("L2C-310 errat%s", n > 1 ? "a" : "um"); for (i = 0; i < n; i++) pr_cont(" %s", errata[i]); @@ -774,6 +775,11 @@ static const struct l2c_init_data l2c310_init_fns __initconst = { }, }; +void l2c310_flush_all(void) +{ + l2c210_flush_all(); +}; + static int __init __l2c_init(const struct l2c_init_data *data, u32 aux_val, u32 aux_mask, u32 cache_id) { @@ -876,6 +882,8 @@ static int __init __l2c_init(const struct l2c_init_data *data, pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n", data->type, cache_id, aux); + litmus_setup_lockdown(l2x0_base, cache_id); + return 0; } diff --git a/arch/x86/syscalls/syscall_32.tbl b/arch/x86/syscalls/syscall_32.tbl index a0ec8abc51f3..af5e3ccf8b31 100644 --- a/arch/x86/syscalls/syscall_32.tbl +++ b/arch/x86/syscalls/syscall_32.tbl @@ -378,3 +378,4 @@ 369 i386 release_ts sys_release_ts 370 i386 null_call sys_null_call 371 i386 get_current_budget sys_get_current_budget +372 i386 test_call sys_test_call \ No newline at end of file diff --git a/arch/x86/syscalls/syscall_64.tbl b/arch/x86/syscalls/syscall_64.tbl index 04f5b7483db3..e87042d413e9 100644 --- a/arch/x86/syscalls/syscall_64.tbl +++ b/arch/x86/syscalls/syscall_64.tbl @@ -343,6 +343,7 @@ 361 common release_ts sys_release_ts 362 common null_call sys_null_call 363 common get_current_budget sys_get_current_budget +364 common test_call sys_test_call # diff --git a/include/litmus/cache_proc.h b/include/litmus/cache_proc.h new file mode 100644 index 000000000000..586224118435 --- /dev/null +++ b/include/litmus/cache_proc.h @@ -0,0 +1,11 @@ +#ifndef LITMUS_CACHE_PROC_H +#define LITMUS_CACHE_PROC_H + +#ifdef __KERNEL__ + +void litmus_setup_lockdown(void __iomem*, u32); + +#endif + +#endif + diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h index 570b1f54d534..43527ee2f6b6 100644 --- a/include/litmus/unistd_32.h +++ b/include/litmus/unistd_32.h @@ -18,5 +18,6 @@ #define __NR_release_ts __LSC(10) #define __NR_null_call __LSC(11) #define __NR_get_current_budget __LSC(12) +#define __NR_test_call __LSC(13) -#define NR_litmus_syscalls 13 +#define NR_litmus_syscalls 14 diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h index 3096bf2f2798..1a741bd3e5da 100644 --- a/include/litmus/unistd_64.h +++ b/include/litmus/unistd_64.h @@ -31,5 +31,7 @@ __SYSCALL(__NR_release_ts, sys_release_ts) __SYSCALL(__NR_null_call, sys_null_call) #define __NR_get_current_budget __LSC(12) __SYSCALL(____NR_get_current_budget, sys_get_current_budget) +#define __NR_test_call __LSC(13) +__SYSCALL(__NR_test_call, sys_test_call) -#define NR_litmus_syscalls 13 +#define NR_litmus_syscalls 14 diff --git a/litmus/Makefile b/litmus/Makefile index 7970cd55e7fd..f80a3c0d05aa 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -21,7 +21,8 @@ obj-y = sched_plugin.o litmus.o \ uncachedev.o \ sched_gsn_edf.o \ sched_psn_edf.o \ - sched_pfp.o + sched_pfp.o \ + cache_proc.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o diff --git a/litmus/cache_proc.c b/litmus/cache_proc.c new file mode 100644 index 000000000000..f5879f32232a --- /dev/null +++ b/litmus/cache_proc.c @@ -0,0 +1,965 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + + +#define UNLOCK_ALL 0x00000000 /* allocation in any way */ +#define LOCK_ALL (~UNLOCK_ALL) +#define MAX_NR_WAYS 16 +#define MAX_NR_COLORS 16 + +void mem_lock(u32 lock_val, int cpu); + +/* + * unlocked_way[i] : allocation can occur in way i + * + * 0 = allocation can occur in the corresponding way + * 1 = allocation cannot occur in the corresponding way + */ +u32 unlocked_way[MAX_NR_WAYS] = { + 0xFFFFFFFE, /* way 0 unlocked */ + 0xFFFFFFFD, + 0xFFFFFFFB, + 0xFFFFFFF7, + 0xFFFFFFEF, /* way 4 unlocked */ + 0xFFFFFFDF, + 0xFFFFFFBF, + 0xFFFFFF7F, + 0xFFFFFEFF, /* way 8 unlocked */ + 0xFFFFFDFF, + 0xFFFFFBFF, + 0xFFFFF7FF, + 0xFFFFEFFF, /* way 12 unlocked */ + 0xFFFFDFFF, + 0xFFFFBFFF, + 0xFFFF7FFF, +}; + +u32 nr_unlocked_way[MAX_NR_WAYS+1] = { + 0x0000FFFF, /* all ways are locked. usable = 0*/ + 0x0000FFFE, /* way ~0 unlocked. usable = 1 */ + 0x0000FFFC, + 0x0000FFF8, + 0x0000FFF0, + 0x0000FFE0, + 0x0000FFC0, + 0x0000FF80, + 0x0000FF00, + 0x0000FE00, + 0x0000FC00, + 0x0000F800, + 0x0000F000, + 0x0000E000, + 0x0000C000, + 0x00008000, + 0x00000000, /* way ~15 unlocked. usable = 16 */ +}; + +u32 way_partition[4] = { + 0xfffffff0, /* cpu0 */ + 0xffffff0f, /* cpu1 */ + 0xfffff0ff, /* cpu2 */ + 0xffff0fff, /* cpu3 */ +}; + +u32 way_partitions[9] = { + 0xffff0003, /* cpu0 A */ + 0xffff0003, /* cpu0 B */ + 0xffff000C, /* cpu1 A */ + 0xffff000C, /* cpu1 B */ + 0xffff0030, /* cpu2 A */ + 0xffff0030, /* cpu2 B */ + 0xffff00C0, /* cpu3 A */ + 0xffff00C0, /* cpu3 B */ + 0xffffff00, /* lv C */ +}; + +u32 prev_lockdown_d_reg[5] = { + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x000000FF, /* share with level-C */ +}; + +u32 prev_lockdown_i_reg[5] = { + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x000000FF, /* share with level-C */ +}; + +u32 prev_lbm_i_reg[8] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + +u32 prev_lbm_d_reg[8] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + +static void __iomem *cache_base; +static void __iomem *lockreg_d; +static void __iomem *lockreg_i; + +static u32 cache_id; + +struct mutex actlr_mutex; +struct mutex l2x0_prefetch_mutex; +struct mutex lockdown_proc; +static u32 way_partition_min; +static u32 way_partition_max; + +static int zero = 0; +static int one = 1; + +static int l1_prefetch_proc; +static int l2_prefetch_hint_proc; +static int l2_double_linefill_proc; +static int l2_data_prefetch_proc; + +u32 lockdown_reg[9] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + + +#define ld_d_reg(cpu) ({ int __cpu = cpu; \ + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_D_BASE + \ + __cpu * L2X0_LOCKDOWN_STRIDE; __v; }) +#define ld_i_reg(cpu) ({ int __cpu = cpu; \ + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_I_BASE + \ + __cpu * L2X0_LOCKDOWN_STRIDE; __v; }) + +int lock_all; +int nr_lockregs; +static raw_spinlock_t cache_lock; +static raw_spinlock_t prefetch_lock; + +extern void l2c310_flush_all(void); + +static inline void cache_wait_way(void __iomem *reg, unsigned long mask) +{ + /* wait for cache operation by line or way to complete */ + while (readl_relaxed(reg) & mask) + cpu_relax(); +} + +#ifdef CONFIG_CACHE_L2X0 +static inline void cache_wait(void __iomem *reg, unsigned long mask) +{ + /* cache operations by line are atomic on PL310 */ +} +#else +#define cache_wait cache_wait_way +#endif + +static inline void cache_sync(void) +{ + void __iomem *base = cache_base; + + writel_relaxed(0, base + L2X0_CACHE_SYNC); + cache_wait(base + L2X0_CACHE_SYNC, 1); +} + +static void print_lockdown_registers(int cpu) +{ + int i; + //for (i = 0; i < nr_lockregs; i++) { + for (i = 0; i < 4; i++) { + printk("P%d Lockdown Data CPU %2d: 0x%04x\n", cpu, + i, readl_relaxed(ld_d_reg(i))); + printk("P%d Lockdown Inst CPU %2d: 0x%04x\n", cpu, + i, readl_relaxed(ld_i_reg(i))); + } +} + +static void test_lockdown(void *ignore) +{ + int i, cpu; + + cpu = smp_processor_id(); + printk("Start lockdown test on CPU %d.\n", cpu); + + for (i = 0; i < nr_lockregs; i++) { + printk("CPU %2d data reg: 0x%8p\n", i, ld_d_reg(i)); + printk("CPU %2d inst reg: 0x%8p\n", i, ld_i_reg(i)); + } + + printk("Lockdown initial state:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(1, ld_d_reg(i)); + writel_relaxed(2, ld_i_reg(i)); + } + printk("Lockdown all data=1 instr=2:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed((1 << i), ld_d_reg(i)); + writel_relaxed(((1 << 8) >> i), ld_i_reg(i)); + } + printk("Lockdown varies:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(UNLOCK_ALL, ld_d_reg(i)); + writel_relaxed(UNLOCK_ALL, ld_i_reg(i)); + } + printk("Lockdown all zero:\n"); + print_lockdown_registers(cpu); + + printk("End lockdown test.\n"); +} + +void litmus_setup_lockdown(void __iomem *base, u32 id) +{ + cache_base = base; + cache_id = id; + lockreg_d = cache_base + L2X0_LOCKDOWN_WAY_D_BASE; + lockreg_i = cache_base + L2X0_LOCKDOWN_WAY_I_BASE; + + if (L2X0_CACHE_ID_PART_L310 == (cache_id & L2X0_CACHE_ID_PART_MASK)) { + nr_lockregs = 8; + } else { + printk("Unknown cache ID!\n"); + nr_lockregs = 1; + } + + mutex_init(&actlr_mutex); + mutex_init(&l2x0_prefetch_mutex); + mutex_init(&lockdown_proc); + raw_spin_lock_init(&cache_lock); + raw_spin_lock_init(&prefetch_lock); + + test_lockdown(NULL); +} + +int way_partition_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + unsigned long flags; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + printk("Way-partition settings:\n"); + for (i = 0; i < 9; i++) { + printk("0x%08X\n", way_partitions[i]); + } + for (i = 0; i < 4; i++) { + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + + local_irq_save(flags); + print_lockdown_registers(smp_processor_id()); + l2c310_flush_all(); + local_irq_restore(flags); +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lock_all_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + unsigned long flags; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write && lock_all == 1) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } +/* + for (i = 0; i < nr_lockregs; i++) { + barrier(); + mem_lock(LOCK_ALL, i); + barrier(); + //writel_relaxed(nr_unlocked_way[0], ld_d_reg(i)); + //writel_relaxed(nr_unlocked_way[0], ld_i_reg(i)); + } +*/ + } + if (write && lock_all == 0) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } +/* + for (i = 0; i < nr_lockregs; i++) { + barrier(); + mem_lock(UNLOCK_ALL, i); + barrier(); + //writel_relaxed(nr_unlocked_way[16], ld_d_reg(i)); + //writel_relaxed(nr_unlocked_way[16], ld_i_reg(i)); + } +*/ + } + printk("LOCK_ALL HANDLER\n"); + local_irq_save(flags); + print_lockdown_registers(smp_processor_id()); + l2c310_flush_all(); + local_irq_restore(flags); +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +void cache_lockdown(u32 lock_val, int cpu) +{ + //unsigned long flags; + //raw_spin_lock_irqsave(&cache_lock, flags); + + __asm__ __volatile__ ( +" str %[lockval], [%[dcachereg]]\n" +" str %[lockval], [%[icachereg]]\n" + : + : [dcachereg] "r" (ld_d_reg(cpu)), + [icachereg] "r" (ld_i_reg(cpu)), + [lockval] "r" (lock_val) + : "cc"); + + //raw_spin_unlock_irqrestore(&cache_lock, flags); +} + +int lockdown_reg_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lockdown_global_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +/* Operate on the Cortex-A9's ACTLR register */ +#define ACTLR_L2_PREFETCH_HINT (1 << 1) +#define ACTLR_L1_PREFETCH (1 << 2) + +/* + * Change the ACTLR. + * @mode - If 1 (0), set (clear) the bit given in @mask in the ACTLR. + * @mask - A mask in which one bit is set to operate on the ACTLR. + */ +static void actlr_change(int mode, int mask) +{ + u32 orig_value, new_value, reread_value; + + if (0 != mode && 1 != mode) { + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n", + __FUNCTION__); + return; + } + + /* get the original value */ + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (orig_value)); + + if (0 == mode) + new_value = orig_value & ~(mask); + else + new_value = orig_value | mask; + + asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (new_value)); + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (reread_value)); + + printk("ACTLR: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n", + orig_value, new_value, reread_value); +} + +int litmus_l1_prefetch_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&actlr_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + + if (!ret && write) { + mode = *((int*)table->data); + actlr_change(mode, ACTLR_L1_PREFETCH); + } + mutex_unlock(&actlr_mutex); + + return ret; +} + +int litmus_l2_prefetch_hint_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&actlr_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + actlr_change(mode, ACTLR_L2_PREFETCH_HINT); + } + mutex_unlock(&actlr_mutex); + + return ret; +} + + +/* Operate on the PL-310's Prefetch Control Register, L310_PREFETCH_CTRL */ +#define L2X0_PREFETCH_DOUBLE_LINEFILL (1 << 30) +#define L2X0_PREFETCH_INST_PREFETCH (1 << 29) +#define L2X0_PREFETCH_DATA_PREFETCH (1 << 28) +static void l2x0_prefetch_change(int mode, int mask) +{ + u32 orig_value, new_value, reread_value; + + if (0 != mode && 1 != mode) { + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n", + __FUNCTION__); + return; + } + + orig_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL); + + if (0 == mode) + new_value = orig_value & ~(mask); + else + new_value = orig_value | mask; + + writel_relaxed(new_value, cache_base + L310_PREFETCH_CTRL); + reread_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL); + + printk("l2x0 prefetch: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n", + orig_value, new_value, reread_value); +} + +int litmus_l2_double_linefill_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&l2x0_prefetch_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + l2x0_prefetch_change(mode, L2X0_PREFETCH_DOUBLE_LINEFILL); + } + mutex_unlock(&l2x0_prefetch_mutex); + + return ret; +} + +int litmus_l2_data_prefetch_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&l2x0_prefetch_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + l2x0_prefetch_change(mode, L2X0_PREFETCH_DATA_PREFETCH|L2X0_PREFETCH_INST_PREFETCH); + } + mutex_unlock(&l2x0_prefetch_mutex); + + return ret; +} + +int do_perf_test_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +int setup_flusher_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +static struct ctl_table cache_table[] = +{ + { + .procname = "C0_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[0], + .maxlen = sizeof(way_partitions[0]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C0_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[1], + .maxlen = sizeof(way_partitions[1]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C1_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[2], + .maxlen = sizeof(way_partitions[2]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C1_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[3], + .maxlen = sizeof(way_partitions[3]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C2_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[4], + .maxlen = sizeof(way_partitions[4]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C2_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[5], + .maxlen = sizeof(way_partitions[5]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C3_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[6], + .maxlen = sizeof(way_partitions[6]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C3_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[7], + .maxlen = sizeof(way_partitions[7]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "Call_LC_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[8], + .maxlen = sizeof(way_partitions[8]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lock_all", + .mode = 0666, + .proc_handler = lock_all_handler, + .data = &lock_all, + .maxlen = sizeof(lock_all), + .extra1 = &zero, + .extra2 = &one, + }, + { + .procname = "l1_prefetch", + .mode = 0644, + .proc_handler = litmus_l1_prefetch_proc_handler, + .data = &l1_prefetch_proc, + .maxlen = sizeof(l1_prefetch_proc), + }, + { + .procname = "l2_prefetch_hint", + .mode = 0644, + .proc_handler = litmus_l2_prefetch_hint_proc_handler, + .data = &l2_prefetch_hint_proc, + .maxlen = sizeof(l2_prefetch_hint_proc), + }, + { + .procname = "l2_double_linefill", + .mode = 0644, + .proc_handler = litmus_l2_double_linefill_proc_handler, + .data = &l2_double_linefill_proc, + .maxlen = sizeof(l2_double_linefill_proc), + }, + { + .procname = "l2_data_prefetch", + .mode = 0644, + .proc_handler = litmus_l2_data_prefetch_proc_handler, + .data = &l2_data_prefetch_proc, + .maxlen = sizeof(l2_data_prefetch_proc), + }, + { + .procname = "do_perf_test", + .mode = 0644, + .proc_handler = do_perf_test_proc_handler, + }, + { + .procname = "lockdown_reg_0", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[0], + .maxlen = sizeof(lockdown_reg[0]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_1", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[1], + .maxlen = sizeof(lockdown_reg[1]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_2", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[2], + .maxlen = sizeof(lockdown_reg[2]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_3", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[3], + .maxlen = sizeof(lockdown_reg[3]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_regs", + .mode = 0644, + .proc_handler = lockdown_global_handler, + .data = &lockdown_reg[8], + .maxlen = sizeof(lockdown_reg[8]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { } +}; + +static struct ctl_table litmus_dir_table[] = { + { + .procname = "litmus", + .mode = 0555, + .child = cache_table, + }, + { } +}; + +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end) +{ + u32 v = 0; + + __asm__ __volatile__ ( +" .align 5\n" +" str %[lockval], [%[cachereg]]\n" +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n" +" cmp %[end], %[addr] @ subtracts addr from end\n" +" bgt 1b\n @ read more, if necessary\n" + : [addr] "+r" (start), + [val] "+r" (v) + : [end] "r" (end), +#ifdef CONFIG_CACHE_L2X0 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())), +#else + [cachereg] "r" (lockreg_d), +#endif + [lockval] "r" (lock_val) + : "cc"); + + return v; +} + + +/* + * Prefetch by reading the first word of each cache line in a page. + * + * @lockdown_reg: address of the lockdown register to write + * @lock_val: value to be written to @lockdown_reg + * @unlock_val: will unlock the cache to this value + * @addr: start address to be prefetched + * @end_addr: end address to prefetch (exclusive) + * + * Assumes: addr < end_addr AND addr != end_addr + */ +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end) +{ +#ifndef CONFIG_CACHE_L2X0 + unsigned long flags; +#endif + u32 v = 0; + +#ifndef CONFIG_CACHE_L2X0 + raw_spin_lock_irqsave(&prefetch_lock, flags); +#endif + + __asm__ __volatile__ ( +" .align 5\n" +" str %[lockval], [%[cachereg]]\n" +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n" +" cmp %[end], %[addr] @ subtracts addr from end\n" +" bgt 1b\n @ read more, if necessary\n" +" str %[unlockval], [%[cachereg]]\n" + : [addr] "+r" (start), + [val] "+r" (v) + : [end] "r" (end), +#ifdef CONFIG_CACHE_L2X0 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())), +#else + [cachereg] "r" (lockreg_d), +#endif + [lockval] "r" (lock_val), + [unlockval] "r" (unlock_val) + : "cc"); + +#ifndef CONFIG_CACHE_L2X0 + raw_spin_unlock_irqrestore(&prefetch_lock, flags); +#endif + + return v; +} + +static long update_timeval(struct timespec lhs, struct timespec rhs) +{ + long val; + struct timespec ts; + + ts = timespec_sub(rhs, lhs); + val = ts.tv_sec*NSEC_PER_SEC + ts.tv_nsec; + + return val; +} + +extern void v7_flush_kern_dcache_area(void *, size_t); +extern void v7_flush_kern_cache_all(void); +/* + * Ensure that this page is not in the L1 or L2 cache. + * Since the L1 cache is VIPT and the L2 cache is PIPT, we can use either the + * kernel or user vaddr. + */ +void color_flush_page(void *vaddr, size_t size) +{ + //v7_flush_kern_dcache_area(vaddr, size); + v7_flush_kern_cache_all(); +} + +#define TRIALS 1000 + +static int perf_test(void) { + struct timespec before, after; + struct page *page; + void *vaddr; + u32 *data; + long time, flush_time; + int i, num_pages = 1; + unsigned int order = 4; + + for (i = 0; i < order; i++) { + num_pages = num_pages*2; + } + + printk("Number of pages: %d\n", num_pages); + //page = alloc_page(__GFP_MOVABLE); + page = alloc_pages(__GFP_MOVABLE, order); + if (!page) { + printk(KERN_WARNING "No memory\n"); + return -ENOMEM; + } + + vaddr = page_address(page); + if (!vaddr) + printk(KERN_WARNING "%s: vaddr is null\n", __FUNCTION__); + data = (u32*) vaddr; + + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_flush_page(vaddr, PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for flushes without re-reading: %ld\n", time / TRIALS); + flush_time = time / TRIALS; + + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + + barrier(); + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for read from cache: %ld\n", time / TRIALS); + + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + color_flush_page(vaddr, PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for read from mem: %ld (%ld)\n", time / TRIALS - flush_time, time / TRIALS); + + // write in locked way + color_read_in_mem_lock(nr_unlocked_way[2], LOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + for (i = 0; i < PAGE_SIZE*num_pages/sizeof(u32); i++) { + data[i] = i%63353; + } + // read + barrier(); + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_read_in_mem(unlocked_way[0], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for read in after write: %ld\n", time / TRIALS); + + + //free_page((unsigned long)vaddr); + free_pages((unsigned long)vaddr, order); + + return 0; +} + +int do_perf_test_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = 0; + + if (write) { + ret = perf_test(); + } + + return ret; +} + +static struct ctl_table_header *litmus_sysctls; + +static int __init litmus_sysctl_init(void) +{ + int ret = 0; + + printk(KERN_INFO "Registering LITMUS^RT proc sysctl.\n"); + litmus_sysctls = register_sysctl_table(litmus_dir_table); + if (!litmus_sysctls) { + printk(KERN_WARNING "Could not register LITMUS^RT sysctl.\n"); + ret = -EFAULT; + goto out; + } + + way_partition_min = 0x00000000; + way_partition_max = 0x0000FFFF; + +out: + return ret; +} + +module_init(litmus_sysctl_init); diff --git a/litmus/litmus.c b/litmus/litmus.c index db5ce0e9c76e..27efb22d1d2f 100644 --- a/litmus/litmus.c +++ b/litmus/litmus.c @@ -31,6 +31,8 @@ #include #endif +extern void l2c310_flush_all(void); + /* Number of RT tasks that exist in the system */ atomic_t rt_task_count = ATOMIC_INIT(0); @@ -314,6 +316,43 @@ asmlinkage long sys_null_call(cycles_t __user *ts) return ret; } +/* sys_test_call() is a test system call for developing */ +asmlinkage long sys_test_call(unsigned int param) +{ + long ret = 0; + unsigned long flags; + struct vm_area_struct *vma_itr = NULL; + + TRACE_CUR("test_call param = %d\n", param); + + down_read(¤t->mm->mmap_sem); + vma_itr = current->mm->mmap; + while (vma_itr != NULL) { + printk(KERN_INFO "vm_start : %lx\n", vma_itr->vm_start); + printk(KERN_INFO "vm_end : %lx\n", vma_itr->vm_end); + printk(KERN_INFO "vm_flags : %lx\n", vma_itr->vm_flags); + printk(KERN_INFO "vm_prot : %x\n", pgprot_val(vma_itr->vm_page_prot)); + printk(KERN_INFO "VM_SHARED? %ld\n", vma_itr->vm_flags & VM_SHARED); + if (vma_itr->vm_file) { + struct file *fp = vma_itr->vm_file; + unsigned long fcount = atomic_long_read(&(fp->f_count)); + printk(KERN_INFO "f_count : %ld\n", fcount); + if (fcount > 1) { + vma_itr->vm_page_prot = pgprot_noncached(vma_itr->vm_page_prot); + } + } + printk(KERN_INFO "vm_prot2 : %x\n", pgprot_val(vma_itr->vm_page_prot)); + vma_itr = vma_itr->vm_next; + } + up_read(¤t->mm->mmap_sem); + + local_irq_save(flags); + l2c310_flush_all(); + local_irq_restore(flags); + + return ret; +} + /* p is a real-time task. Re-init its state as a best-effort task. */ static void reinit_litmus_state(struct task_struct* p, int restore) { -- cgit v1.2.2