diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2018-06-04 22:17:47 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2018-06-04 22:17:47 -0400 |
commit | 0afe832e55a70f4bc5e725db400779b4f620290c (patch) | |
tree | a656975173d7ed2cc65ef3cf38c16be48c81b85a | |
parent | 42964c6f6261f6b44bb43cabc93ac9e39b5a9a54 (diff) | |
parent | 844ea8f62619ac2a5e47d13e3449ac197067b414 (diff) |
Merge branch 'x86-cleanups-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 cleanups from Ingo Molnar:
"Misc cleanups"
* 'x86-cleanups-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
x86/apm: Fix spelling mistake: "caculate" -> "calculate"
x86/mtrr: Rename main.c to mtrr.c and remove duplicate prefixes
x86: Remove pr_fmt duplicate logging prefixes
x86/early-quirks: Rename duplicate define of dev_err
x86/bpf: Clean up non-standard comments, to make the code more readable
-rw-r--r-- | arch/x86/events/amd/ibs.c | 2 | ||||
-rw-r--r-- | arch/x86/kernel/apm_32.c | 2 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/mtrr/Makefile | 2 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/mtrr/mtrr.c (renamed from arch/x86/kernel/cpu/mtrr/main.c) | 33 | ||||
-rw-r--r-- | arch/x86/kernel/e820.c | 32 | ||||
-rw-r--r-- | arch/x86/kernel/early-quirks.c | 8 | ||||
-rw-r--r-- | arch/x86/kernel/hpet.c | 5 | ||||
-rw-r--r-- | arch/x86/kernel/uprobes.c | 4 | ||||
-rw-r--r-- | arch/x86/mm/numa.c | 22 | ||||
-rw-r--r-- | arch/x86/net/bpf_jit_comp.c | 233 |
10 files changed, 188 insertions, 155 deletions
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 786fd875de92..4b98101209a1 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c | |||
@@ -889,7 +889,7 @@ static void force_ibs_eilvt_setup(void) | |||
889 | if (!ibs_eilvt_valid()) | 889 | if (!ibs_eilvt_valid()) |
890 | goto out; | 890 | goto out; |
891 | 891 | ||
892 | pr_info("IBS: LVT offset %d assigned\n", offset); | 892 | pr_info("LVT offset %d assigned\n", offset); |
893 | 893 | ||
894 | return; | 894 | return; |
895 | out: | 895 | out: |
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index cadeafabf167..5d0de79fdab0 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c | |||
@@ -2433,7 +2433,7 @@ MODULE_PARM_DESC(idle_threshold, | |||
2433 | "System idle percentage above which to make APM BIOS idle calls"); | 2433 | "System idle percentage above which to make APM BIOS idle calls"); |
2434 | module_param(idle_period, int, 0444); | 2434 | module_param(idle_period, int, 0444); |
2435 | MODULE_PARM_DESC(idle_period, | 2435 | MODULE_PARM_DESC(idle_period, |
2436 | "Period (in sec/100) over which to caculate the idle percentage"); | 2436 | "Period (in sec/100) over which to calculate the idle percentage"); |
2437 | module_param(smp, bool, 0444); | 2437 | module_param(smp, bool, 0444); |
2438 | MODULE_PARM_DESC(smp, | 2438 | MODULE_PARM_DESC(smp, |
2439 | "Set this to enable APM use on an SMP platform. Use with caution on older systems"); | 2439 | "Set this to enable APM use on an SMP platform. Use with caution on older systems"); |
diff --git a/arch/x86/kernel/cpu/mtrr/Makefile b/arch/x86/kernel/cpu/mtrr/Makefile index ad9e5ed81181..2ad9107ee980 100644 --- a/arch/x86/kernel/cpu/mtrr/Makefile +++ b/arch/x86/kernel/cpu/mtrr/Makefile | |||
@@ -1,3 +1,3 @@ | |||
1 | obj-y := main.o if.o generic.o cleanup.o | 1 | obj-y := mtrr.o if.o generic.o cleanup.o |
2 | obj-$(CONFIG_X86_32) += amd.o cyrix.o centaur.o | 2 | obj-$(CONFIG_X86_32) += amd.o cyrix.o centaur.o |
3 | 3 | ||
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/mtrr.c index 3ea0047beb40..9a19c800fe40 100644 --- a/arch/x86/kernel/cpu/mtrr/main.c +++ b/arch/x86/kernel/cpu/mtrr/mtrr.c | |||
@@ -101,7 +101,7 @@ static int have_wrcomb(void) | |||
101 | if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS && | 101 | if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS && |
102 | dev->device == PCI_DEVICE_ID_SERVERWORKS_LE && | 102 | dev->device == PCI_DEVICE_ID_SERVERWORKS_LE && |
103 | dev->revision <= 5) { | 103 | dev->revision <= 5) { |
104 | pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n"); | 104 | pr_info("Serverworks LE rev < 6 detected. Write-combining disabled.\n"); |
105 | pci_dev_put(dev); | 105 | pci_dev_put(dev); |
106 | return 0; | 106 | return 0; |
107 | } | 107 | } |
@@ -111,7 +111,7 @@ static int have_wrcomb(void) | |||
111 | */ | 111 | */ |
112 | if (dev->vendor == PCI_VENDOR_ID_INTEL && | 112 | if (dev->vendor == PCI_VENDOR_ID_INTEL && |
113 | dev->device == PCI_DEVICE_ID_INTEL_82451NX) { | 113 | dev->device == PCI_DEVICE_ID_INTEL_82451NX) { |
114 | pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n"); | 114 | pr_info("Intel 450NX MMC detected. Write-combining disabled.\n"); |
115 | pci_dev_put(dev); | 115 | pci_dev_put(dev); |
116 | return 0; | 116 | return 0; |
117 | } | 117 | } |
@@ -313,24 +313,24 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
313 | return error; | 313 | return error; |
314 | 314 | ||
315 | if (type >= MTRR_NUM_TYPES) { | 315 | if (type >= MTRR_NUM_TYPES) { |
316 | pr_warn("mtrr: type: %u invalid\n", type); | 316 | pr_warn("type: %u invalid\n", type); |
317 | return -EINVAL; | 317 | return -EINVAL; |
318 | } | 318 | } |
319 | 319 | ||
320 | /* If the type is WC, check that this processor supports it */ | 320 | /* If the type is WC, check that this processor supports it */ |
321 | if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) { | 321 | if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) { |
322 | pr_warn("mtrr: your processor doesn't support write-combining\n"); | 322 | pr_warn("your processor doesn't support write-combining\n"); |
323 | return -ENOSYS; | 323 | return -ENOSYS; |
324 | } | 324 | } |
325 | 325 | ||
326 | if (!size) { | 326 | if (!size) { |
327 | pr_warn("mtrr: zero sized request\n"); | 327 | pr_warn("zero sized request\n"); |
328 | return -EINVAL; | 328 | return -EINVAL; |
329 | } | 329 | } |
330 | 330 | ||
331 | if ((base | (base + size - 1)) >> | 331 | if ((base | (base + size - 1)) >> |
332 | (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) { | 332 | (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) { |
333 | pr_warn("mtrr: base or size exceeds the MTRR width\n"); | 333 | pr_warn("base or size exceeds the MTRR width\n"); |
334 | return -EINVAL; | 334 | return -EINVAL; |
335 | } | 335 | } |
336 | 336 | ||
@@ -361,8 +361,7 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
361 | } else if (types_compatible(type, ltype)) | 361 | } else if (types_compatible(type, ltype)) |
362 | continue; | 362 | continue; |
363 | } | 363 | } |
364 | pr_warn("mtrr: 0x%lx000,0x%lx000 overlaps existing" | 364 | pr_warn("0x%lx000,0x%lx000 overlaps existing 0x%lx000,0x%lx000\n", base, size, lbase, |
365 | " 0x%lx000,0x%lx000\n", base, size, lbase, | ||
366 | lsize); | 365 | lsize); |
367 | goto out; | 366 | goto out; |
368 | } | 367 | } |
@@ -370,7 +369,7 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
370 | if (ltype != type) { | 369 | if (ltype != type) { |
371 | if (types_compatible(type, ltype)) | 370 | if (types_compatible(type, ltype)) |
372 | continue; | 371 | continue; |
373 | pr_warn("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n", | 372 | pr_warn("type mismatch for %lx000,%lx000 old: %s new: %s\n", |
374 | base, size, mtrr_attrib_to_str(ltype), | 373 | base, size, mtrr_attrib_to_str(ltype), |
375 | mtrr_attrib_to_str(type)); | 374 | mtrr_attrib_to_str(type)); |
376 | goto out; | 375 | goto out; |
@@ -396,7 +395,7 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
396 | } | 395 | } |
397 | } | 396 | } |
398 | } else { | 397 | } else { |
399 | pr_info("mtrr: no more MTRRs available\n"); | 398 | pr_info("no more MTRRs available\n"); |
400 | } | 399 | } |
401 | error = i; | 400 | error = i; |
402 | out: | 401 | out: |
@@ -408,8 +407,8 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
408 | static int mtrr_check(unsigned long base, unsigned long size) | 407 | static int mtrr_check(unsigned long base, unsigned long size) |
409 | { | 408 | { |
410 | if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) { | 409 | if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) { |
411 | pr_warn("mtrr: size and base must be multiples of 4 kiB\n"); | 410 | pr_warn("size and base must be multiples of 4 kiB\n"); |
412 | pr_debug("mtrr: size: 0x%lx base: 0x%lx\n", size, base); | 411 | pr_debug("size: 0x%lx base: 0x%lx\n", size, base); |
413 | dump_stack(); | 412 | dump_stack(); |
414 | return -1; | 413 | return -1; |
415 | } | 414 | } |
@@ -500,22 +499,22 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size) | |||
500 | } | 499 | } |
501 | } | 500 | } |
502 | if (reg < 0) { | 501 | if (reg < 0) { |
503 | pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n", | 502 | pr_debug("no MTRR for %lx000,%lx000 found\n", |
504 | base, size); | 503 | base, size); |
505 | goto out; | 504 | goto out; |
506 | } | 505 | } |
507 | } | 506 | } |
508 | if (reg >= max) { | 507 | if (reg >= max) { |
509 | pr_warn("mtrr: register: %d too big\n", reg); | 508 | pr_warn("register: %d too big\n", reg); |
510 | goto out; | 509 | goto out; |
511 | } | 510 | } |
512 | mtrr_if->get(reg, &lbase, &lsize, <ype); | 511 | mtrr_if->get(reg, &lbase, &lsize, <ype); |
513 | if (lsize < 1) { | 512 | if (lsize < 1) { |
514 | pr_warn("mtrr: MTRR %d not used\n", reg); | 513 | pr_warn("MTRR %d not used\n", reg); |
515 | goto out; | 514 | goto out; |
516 | } | 515 | } |
517 | if (mtrr_usage_table[reg] < 1) { | 516 | if (mtrr_usage_table[reg] < 1) { |
518 | pr_warn("mtrr: reg: %d has count=0\n", reg); | 517 | pr_warn("reg: %d has count=0\n", reg); |
519 | goto out; | 518 | goto out; |
520 | } | 519 | } |
521 | if (--mtrr_usage_table[reg] < 1) | 520 | if (--mtrr_usage_table[reg] < 1) |
@@ -776,7 +775,7 @@ void __init mtrr_bp_init(void) | |||
776 | } | 775 | } |
777 | 776 | ||
778 | if (!mtrr_enabled()) { | 777 | if (!mtrr_enabled()) { |
779 | pr_info("MTRR: Disabled\n"); | 778 | pr_info("Disabled\n"); |
780 | 779 | ||
781 | /* | 780 | /* |
782 | * PAT initialization relies on MTRR's rendezvous handler. | 781 | * PAT initialization relies on MTRR's rendezvous handler. |
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 6a2cb1442e05..d1f25c831447 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c | |||
@@ -155,7 +155,8 @@ static void __init __e820__range_add(struct e820_table *table, u64 start, u64 si | |||
155 | int x = table->nr_entries; | 155 | int x = table->nr_entries; |
156 | 156 | ||
157 | if (x >= ARRAY_SIZE(table->entries)) { | 157 | if (x >= ARRAY_SIZE(table->entries)) { |
158 | pr_err("e820: too many entries; ignoring [mem %#010llx-%#010llx]\n", start, start + size - 1); | 158 | pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n", |
159 | start, start + size - 1); | ||
159 | return; | 160 | return; |
160 | } | 161 | } |
161 | 162 | ||
@@ -190,9 +191,10 @@ void __init e820__print_table(char *who) | |||
190 | int i; | 191 | int i; |
191 | 192 | ||
192 | for (i = 0; i < e820_table->nr_entries; i++) { | 193 | for (i = 0; i < e820_table->nr_entries; i++) { |
193 | pr_info("%s: [mem %#018Lx-%#018Lx] ", who, | 194 | pr_info("%s: [mem %#018Lx-%#018Lx] ", |
194 | e820_table->entries[i].addr, | 195 | who, |
195 | e820_table->entries[i].addr + e820_table->entries[i].size - 1); | 196 | e820_table->entries[i].addr, |
197 | e820_table->entries[i].addr + e820_table->entries[i].size - 1); | ||
196 | 198 | ||
197 | e820_print_type(e820_table->entries[i].type); | 199 | e820_print_type(e820_table->entries[i].type); |
198 | pr_cont("\n"); | 200 | pr_cont("\n"); |
@@ -574,7 +576,7 @@ void __init e820__update_table_print(void) | |||
574 | if (e820__update_table(e820_table)) | 576 | if (e820__update_table(e820_table)) |
575 | return; | 577 | return; |
576 | 578 | ||
577 | pr_info("e820: modified physical RAM map:\n"); | 579 | pr_info("modified physical RAM map:\n"); |
578 | e820__print_table("modified"); | 580 | e820__print_table("modified"); |
579 | } | 581 | } |
580 | 582 | ||
@@ -636,9 +638,8 @@ __init void e820__setup_pci_gap(void) | |||
636 | if (!found) { | 638 | if (!found) { |
637 | #ifdef CONFIG_X86_64 | 639 | #ifdef CONFIG_X86_64 |
638 | gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; | 640 | gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; |
639 | pr_err( | 641 | pr_err("Cannot find an available gap in the 32-bit address range\n"); |
640 | "e820: Cannot find an available gap in the 32-bit address range\n" | 642 | pr_err("PCI devices with unassigned 32-bit BARs may not work!\n"); |
641 | "e820: PCI devices with unassigned 32-bit BARs may not work!\n"); | ||
642 | #else | 643 | #else |
643 | gapstart = 0x10000000; | 644 | gapstart = 0x10000000; |
644 | #endif | 645 | #endif |
@@ -649,7 +650,8 @@ __init void e820__setup_pci_gap(void) | |||
649 | */ | 650 | */ |
650 | pci_mem_start = gapstart; | 651 | pci_mem_start = gapstart; |
651 | 652 | ||
652 | pr_info("e820: [mem %#010lx-%#010lx] available for PCI devices\n", gapstart, gapstart + gapsize - 1); | 653 | pr_info("[mem %#010lx-%#010lx] available for PCI devices\n", |
654 | gapstart, gapstart + gapsize - 1); | ||
653 | } | 655 | } |
654 | 656 | ||
655 | /* | 657 | /* |
@@ -711,7 +713,7 @@ void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len) | |||
711 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); | 713 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
712 | 714 | ||
713 | early_memunmap(sdata, data_len); | 715 | early_memunmap(sdata, data_len); |
714 | pr_info("e820: extended physical RAM map:\n"); | 716 | pr_info("extended physical RAM map:\n"); |
715 | e820__print_table("extended"); | 717 | e820__print_table("extended"); |
716 | } | 718 | } |
717 | 719 | ||
@@ -780,7 +782,7 @@ u64 __init e820__memblock_alloc_reserved(u64 size, u64 align) | |||
780 | addr = __memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); | 782 | addr = __memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); |
781 | if (addr) { | 783 | if (addr) { |
782 | e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); | 784 | e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); |
783 | pr_info("e820: update e820_table_kexec for e820__memblock_alloc_reserved()\n"); | 785 | pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); |
784 | e820__update_table_kexec(); | 786 | e820__update_table_kexec(); |
785 | } | 787 | } |
786 | 788 | ||
@@ -830,8 +832,8 @@ static unsigned long __init e820_end_pfn(unsigned long limit_pfn, enum e820_type | |||
830 | if (last_pfn > max_arch_pfn) | 832 | if (last_pfn > max_arch_pfn) |
831 | last_pfn = max_arch_pfn; | 833 | last_pfn = max_arch_pfn; |
832 | 834 | ||
833 | pr_info("e820: last_pfn = %#lx max_arch_pfn = %#lx\n", | 835 | pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n", |
834 | last_pfn, max_arch_pfn); | 836 | last_pfn, max_arch_pfn); |
835 | return last_pfn; | 837 | return last_pfn; |
836 | } | 838 | } |
837 | 839 | ||
@@ -1005,7 +1007,7 @@ void __init e820__finish_early_params(void) | |||
1005 | if (e820__update_table(e820_table) < 0) | 1007 | if (e820__update_table(e820_table) < 0) |
1006 | early_panic("Invalid user supplied memory map"); | 1008 | early_panic("Invalid user supplied memory map"); |
1007 | 1009 | ||
1008 | pr_info("e820: user-defined physical RAM map:\n"); | 1010 | pr_info("user-defined physical RAM map:\n"); |
1009 | e820__print_table("user"); | 1011 | e820__print_table("user"); |
1010 | } | 1012 | } |
1011 | } | 1013 | } |
@@ -1238,7 +1240,7 @@ void __init e820__memory_setup(void) | |||
1238 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); | 1240 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); |
1239 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); | 1241 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
1240 | 1242 | ||
1241 | pr_info("e820: BIOS-provided physical RAM map:\n"); | 1243 | pr_info("BIOS-provided physical RAM map:\n"); |
1242 | e820__print_table(who); | 1244 | e820__print_table(who); |
1243 | } | 1245 | } |
1244 | 1246 | ||
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c index bae0d32e327b..da5d8ac60062 100644 --- a/arch/x86/kernel/early-quirks.c +++ b/arch/x86/kernel/early-quirks.c | |||
@@ -28,8 +28,6 @@ | |||
28 | #include <asm/irq_remapping.h> | 28 | #include <asm/irq_remapping.h> |
29 | #include <asm/early_ioremap.h> | 29 | #include <asm/early_ioremap.h> |
30 | 30 | ||
31 | #define dev_err(msg) pr_err("pci 0000:%02x:%02x.%d: %s", bus, slot, func, msg) | ||
32 | |||
33 | static void __init fix_hypertransport_config(int num, int slot, int func) | 31 | static void __init fix_hypertransport_config(int num, int slot, int func) |
34 | { | 32 | { |
35 | u32 htcfg; | 33 | u32 htcfg; |
@@ -617,7 +615,8 @@ static void __init apple_airport_reset(int bus, int slot, int func) | |||
617 | 615 | ||
618 | pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); | 616 | pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); |
619 | if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { | 617 | if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { |
620 | dev_err("Cannot power up Apple AirPort card\n"); | 618 | pr_err("pci 0000:%02x:%02x.%d: Cannot power up Apple AirPort card\n", |
619 | bus, slot, func); | ||
621 | return; | 620 | return; |
622 | } | 621 | } |
623 | } | 622 | } |
@@ -628,7 +627,8 @@ static void __init apple_airport_reset(int bus, int slot, int func) | |||
628 | 627 | ||
629 | mmio = early_ioremap(addr, BCM4331_MMIO_SIZE); | 628 | mmio = early_ioremap(addr, BCM4331_MMIO_SIZE); |
630 | if (!mmio) { | 629 | if (!mmio) { |
631 | dev_err("Cannot iomap Apple AirPort card\n"); | 630 | pr_err("pci 0000:%02x:%02x.%d: Cannot iomap Apple AirPort card\n", |
631 | bus, slot, func); | ||
632 | return; | 632 | return; |
633 | } | 633 | } |
634 | 634 | ||
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 8ce4212e2b8d..b6be34ee88e9 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c | |||
@@ -975,8 +975,7 @@ int __init hpet_enable(void) | |||
975 | cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY); | 975 | cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY); |
976 | hpet_writel(cfg, HPET_CFG); | 976 | hpet_writel(cfg, HPET_CFG); |
977 | if (cfg) | 977 | if (cfg) |
978 | pr_warn("HPET: Unrecognized bits %#x set in global cfg\n", | 978 | pr_warn("Unrecognized bits %#x set in global cfg\n", cfg); |
979 | cfg); | ||
980 | 979 | ||
981 | for (i = 0; i <= last; ++i) { | 980 | for (i = 0; i <= last; ++i) { |
982 | cfg = hpet_readl(HPET_Tn_CFG(i)); | 981 | cfg = hpet_readl(HPET_Tn_CFG(i)); |
@@ -988,7 +987,7 @@ int __init hpet_enable(void) | |||
988 | | HPET_TN_64BIT_CAP | HPET_TN_32BIT | HPET_TN_ROUTE | 987 | | HPET_TN_64BIT_CAP | HPET_TN_32BIT | HPET_TN_ROUTE |
989 | | HPET_TN_FSB | HPET_TN_FSB_CAP); | 988 | | HPET_TN_FSB | HPET_TN_FSB_CAP); |
990 | if (cfg) | 989 | if (cfg) |
991 | pr_warn("HPET: Unrecognized bits %#x set in cfg#%u\n", | 990 | pr_warn("Unrecognized bits %#x set in cfg#%u\n", |
992 | cfg, i); | 991 | cfg, i); |
993 | } | 992 | } |
994 | hpet_print_config(); | 993 | hpet_print_config(); |
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index c84bb5396958..58d8d800875d 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c | |||
@@ -1083,8 +1083,8 @@ arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs | |||
1083 | return orig_ret_vaddr; | 1083 | return orig_ret_vaddr; |
1084 | 1084 | ||
1085 | if (nleft != rasize) { | 1085 | if (nleft != rasize) { |
1086 | pr_err("uprobe: return address clobbered: pid=%d, %%sp=%#lx, " | 1086 | pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n", |
1087 | "%%ip=%#lx\n", current->pid, regs->sp, regs->ip); | 1087 | current->pid, regs->sp, regs->ip); |
1088 | 1088 | ||
1089 | force_sig_info(SIGSEGV, SEND_SIG_FORCED, current); | 1089 | force_sig_info(SIGSEGV, SEND_SIG_FORCED, current); |
1090 | } | 1090 | } |
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 25504d5aa816..fa150855647c 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c | |||
@@ -136,13 +136,13 @@ static int __init numa_add_memblk_to(int nid, u64 start, u64 end, | |||
136 | 136 | ||
137 | /* whine about and ignore invalid blks */ | 137 | /* whine about and ignore invalid blks */ |
138 | if (start > end || nid < 0 || nid >= MAX_NUMNODES) { | 138 | if (start > end || nid < 0 || nid >= MAX_NUMNODES) { |
139 | pr_warning("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", | 139 | pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", |
140 | nid, start, end - 1); | 140 | nid, start, end - 1); |
141 | return 0; | 141 | return 0; |
142 | } | 142 | } |
143 | 143 | ||
144 | if (mi->nr_blks >= NR_NODE_MEMBLKS) { | 144 | if (mi->nr_blks >= NR_NODE_MEMBLKS) { |
145 | pr_err("NUMA: too many memblk ranges\n"); | 145 | pr_err("too many memblk ranges\n"); |
146 | return -EINVAL; | 146 | return -EINVAL; |
147 | } | 147 | } |
148 | 148 | ||
@@ -267,14 +267,14 @@ int __init numa_cleanup_meminfo(struct numa_meminfo *mi) | |||
267 | */ | 267 | */ |
268 | if (bi->end > bj->start && bi->start < bj->end) { | 268 | if (bi->end > bj->start && bi->start < bj->end) { |
269 | if (bi->nid != bj->nid) { | 269 | if (bi->nid != bj->nid) { |
270 | pr_err("NUMA: node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n", | 270 | pr_err("node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n", |
271 | bi->nid, bi->start, bi->end - 1, | 271 | bi->nid, bi->start, bi->end - 1, |
272 | bj->nid, bj->start, bj->end - 1); | 272 | bj->nid, bj->start, bj->end - 1); |
273 | return -EINVAL; | 273 | return -EINVAL; |
274 | } | 274 | } |
275 | pr_warning("NUMA: Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n", | 275 | pr_warn("Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n", |
276 | bi->nid, bi->start, bi->end - 1, | 276 | bi->nid, bi->start, bi->end - 1, |
277 | bj->start, bj->end - 1); | 277 | bj->start, bj->end - 1); |
278 | } | 278 | } |
279 | 279 | ||
280 | /* | 280 | /* |
@@ -364,7 +364,7 @@ static int __init numa_alloc_distance(void) | |||
364 | phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), | 364 | phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), |
365 | size, PAGE_SIZE); | 365 | size, PAGE_SIZE); |
366 | if (!phys) { | 366 | if (!phys) { |
367 | pr_warning("NUMA: Warning: can't allocate distance table!\n"); | 367 | pr_warn("Warning: can't allocate distance table!\n"); |
368 | /* don't retry until explicitly reset */ | 368 | /* don't retry until explicitly reset */ |
369 | numa_distance = (void *)1LU; | 369 | numa_distance = (void *)1LU; |
370 | return -ENOMEM; | 370 | return -ENOMEM; |
@@ -410,14 +410,14 @@ void __init numa_set_distance(int from, int to, int distance) | |||
410 | 410 | ||
411 | if (from >= numa_distance_cnt || to >= numa_distance_cnt || | 411 | if (from >= numa_distance_cnt || to >= numa_distance_cnt || |
412 | from < 0 || to < 0) { | 412 | from < 0 || to < 0) { |
413 | pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n", | 413 | pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n", |
414 | from, to, distance); | 414 | from, to, distance); |
415 | return; | 415 | return; |
416 | } | 416 | } |
417 | 417 | ||
418 | if ((u8)distance != distance || | 418 | if ((u8)distance != distance || |
419 | (from == to && distance != LOCAL_DISTANCE)) { | 419 | (from == to && distance != LOCAL_DISTANCE)) { |
420 | pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n", | 420 | pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n", |
421 | from, to, distance); | 421 | from, to, distance); |
422 | return; | 422 | return; |
423 | } | 423 | } |
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 263c8453815e..d765acedc05c 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c | |||
@@ -1,4 +1,5 @@ | |||
1 | /* bpf_jit_comp.c : BPF JIT compiler | 1 | /* |
2 | * bpf_jit_comp.c: BPF JIT compiler | ||
2 | * | 3 | * |
3 | * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) | 4 | * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) |
4 | * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com | 5 | * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
@@ -17,7 +18,7 @@ | |||
17 | #include <asm/nospec-branch.h> | 18 | #include <asm/nospec-branch.h> |
18 | 19 | ||
19 | /* | 20 | /* |
20 | * assembly code in arch/x86/net/bpf_jit.S | 21 | * Assembly code in arch/x86/net/bpf_jit.S |
21 | */ | 22 | */ |
22 | extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; | 23 | extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; |
23 | extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[]; | 24 | extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[]; |
@@ -45,14 +46,15 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) | |||
45 | #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) | 46 | #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) |
46 | #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) | 47 | #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) |
47 | #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) | 48 | #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) |
49 | |||
48 | #define EMIT1_off32(b1, off) \ | 50 | #define EMIT1_off32(b1, off) \ |
49 | do {EMIT1(b1); EMIT(off, 4); } while (0) | 51 | do { EMIT1(b1); EMIT(off, 4); } while (0) |
50 | #define EMIT2_off32(b1, b2, off) \ | 52 | #define EMIT2_off32(b1, b2, off) \ |
51 | do {EMIT2(b1, b2); EMIT(off, 4); } while (0) | 53 | do { EMIT2(b1, b2); EMIT(off, 4); } while (0) |
52 | #define EMIT3_off32(b1, b2, b3, off) \ | 54 | #define EMIT3_off32(b1, b2, b3, off) \ |
53 | do {EMIT3(b1, b2, b3); EMIT(off, 4); } while (0) | 55 | do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0) |
54 | #define EMIT4_off32(b1, b2, b3, b4, off) \ | 56 | #define EMIT4_off32(b1, b2, b3, b4, off) \ |
55 | do {EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) | 57 | do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) |
56 | 58 | ||
57 | static bool is_imm8(int value) | 59 | static bool is_imm8(int value) |
58 | { | 60 | { |
@@ -70,9 +72,10 @@ static bool is_uimm32(u64 value) | |||
70 | } | 72 | } |
71 | 73 | ||
72 | /* mov dst, src */ | 74 | /* mov dst, src */ |
73 | #define EMIT_mov(DST, SRC) \ | 75 | #define EMIT_mov(DST, SRC) \ |
74 | do {if (DST != SRC) \ | 76 | do { \ |
75 | EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \ | 77 | if (DST != SRC) \ |
78 | EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \ | ||
76 | } while (0) | 79 | } while (0) |
77 | 80 | ||
78 | static int bpf_size_to_x86_bytes(int bpf_size) | 81 | static int bpf_size_to_x86_bytes(int bpf_size) |
@@ -89,7 +92,8 @@ static int bpf_size_to_x86_bytes(int bpf_size) | |||
89 | return 0; | 92 | return 0; |
90 | } | 93 | } |
91 | 94 | ||
92 | /* list of x86 cond jumps opcodes (. + s8) | 95 | /* |
96 | * List of x86 cond jumps opcodes (. + s8) | ||
93 | * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32) | 97 | * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32) |
94 | */ | 98 | */ |
95 | #define X86_JB 0x72 | 99 | #define X86_JB 0x72 |
@@ -106,35 +110,37 @@ static int bpf_size_to_x86_bytes(int bpf_size) | |||
106 | #define CHOOSE_LOAD_FUNC(K, func) \ | 110 | #define CHOOSE_LOAD_FUNC(K, func) \ |
107 | ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) | 111 | ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) |
108 | 112 | ||
109 | /* pick a register outside of BPF range for JIT internal work */ | 113 | /* Pick a register outside of BPF range for JIT internal work */ |
110 | #define AUX_REG (MAX_BPF_JIT_REG + 1) | 114 | #define AUX_REG (MAX_BPF_JIT_REG + 1) |
111 | 115 | ||
112 | /* The following table maps BPF registers to x64 registers. | 116 | /* |
117 | * The following table maps BPF registers to x86-64 registers. | ||
113 | * | 118 | * |
114 | * x64 register r12 is unused, since if used as base address | 119 | * x86-64 register R12 is unused, since if used as base address |
115 | * register in load/store instructions, it always needs an | 120 | * register in load/store instructions, it always needs an |
116 | * extra byte of encoding and is callee saved. | 121 | * extra byte of encoding and is callee saved. |
117 | * | 122 | * |
118 | * r9 caches skb->len - skb->data_len | 123 | * R9 caches skb->len - skb->data_len |
119 | * r10 caches skb->data, and used for blinding (if enabled) | 124 | * R10 caches skb->data, and used for blinding (if enabled) |
120 | */ | 125 | */ |
121 | static const int reg2hex[] = { | 126 | static const int reg2hex[] = { |
122 | [BPF_REG_0] = 0, /* rax */ | 127 | [BPF_REG_0] = 0, /* RAX */ |
123 | [BPF_REG_1] = 7, /* rdi */ | 128 | [BPF_REG_1] = 7, /* RDI */ |
124 | [BPF_REG_2] = 6, /* rsi */ | 129 | [BPF_REG_2] = 6, /* RSI */ |
125 | [BPF_REG_3] = 2, /* rdx */ | 130 | [BPF_REG_3] = 2, /* RDX */ |
126 | [BPF_REG_4] = 1, /* rcx */ | 131 | [BPF_REG_4] = 1, /* RCX */ |
127 | [BPF_REG_5] = 0, /* r8 */ | 132 | [BPF_REG_5] = 0, /* R8 */ |
128 | [BPF_REG_6] = 3, /* rbx callee saved */ | 133 | [BPF_REG_6] = 3, /* RBX callee saved */ |
129 | [BPF_REG_7] = 5, /* r13 callee saved */ | 134 | [BPF_REG_7] = 5, /* R13 callee saved */ |
130 | [BPF_REG_8] = 6, /* r14 callee saved */ | 135 | [BPF_REG_8] = 6, /* R14 callee saved */ |
131 | [BPF_REG_9] = 7, /* r15 callee saved */ | 136 | [BPF_REG_9] = 7, /* R15 callee saved */ |
132 | [BPF_REG_FP] = 5, /* rbp readonly */ | 137 | [BPF_REG_FP] = 5, /* RBP readonly */ |
133 | [BPF_REG_AX] = 2, /* r10 temp register */ | 138 | [BPF_REG_AX] = 2, /* R10 temp register */ |
134 | [AUX_REG] = 3, /* r11 temp register */ | 139 | [AUX_REG] = 3, /* R11 temp register */ |
135 | }; | 140 | }; |
136 | 141 | ||
137 | /* is_ereg() == true if BPF register 'reg' maps to x64 r8..r15 | 142 | /* |
143 | * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15 | ||
138 | * which need extra byte of encoding. | 144 | * which need extra byte of encoding. |
139 | * rax,rcx,...,rbp have simpler encoding | 145 | * rax,rcx,...,rbp have simpler encoding |
140 | */ | 146 | */ |
@@ -153,7 +159,7 @@ static bool is_axreg(u32 reg) | |||
153 | return reg == BPF_REG_0; | 159 | return reg == BPF_REG_0; |
154 | } | 160 | } |
155 | 161 | ||
156 | /* add modifiers if 'reg' maps to x64 registers r8..r15 */ | 162 | /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */ |
157 | static u8 add_1mod(u8 byte, u32 reg) | 163 | static u8 add_1mod(u8 byte, u32 reg) |
158 | { | 164 | { |
159 | if (is_ereg(reg)) | 165 | if (is_ereg(reg)) |
@@ -170,13 +176,13 @@ static u8 add_2mod(u8 byte, u32 r1, u32 r2) | |||
170 | return byte; | 176 | return byte; |
171 | } | 177 | } |
172 | 178 | ||
173 | /* encode 'dst_reg' register into x64 opcode 'byte' */ | 179 | /* Encode 'dst_reg' register into x86-64 opcode 'byte' */ |
174 | static u8 add_1reg(u8 byte, u32 dst_reg) | 180 | static u8 add_1reg(u8 byte, u32 dst_reg) |
175 | { | 181 | { |
176 | return byte + reg2hex[dst_reg]; | 182 | return byte + reg2hex[dst_reg]; |
177 | } | 183 | } |
178 | 184 | ||
179 | /* encode 'dst_reg' and 'src_reg' registers into x64 opcode 'byte' */ | 185 | /* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */ |
180 | static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) | 186 | static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) |
181 | { | 187 | { |
182 | return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3); | 188 | return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3); |
@@ -184,27 +190,28 @@ static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) | |||
184 | 190 | ||
185 | static void jit_fill_hole(void *area, unsigned int size) | 191 | static void jit_fill_hole(void *area, unsigned int size) |
186 | { | 192 | { |
187 | /* fill whole space with int3 instructions */ | 193 | /* Fill whole space with INT3 instructions */ |
188 | memset(area, 0xcc, size); | 194 | memset(area, 0xcc, size); |
189 | } | 195 | } |
190 | 196 | ||
191 | struct jit_context { | 197 | struct jit_context { |
192 | int cleanup_addr; /* epilogue code offset */ | 198 | int cleanup_addr; /* Epilogue code offset */ |
193 | bool seen_ld_abs; | 199 | bool seen_ld_abs; |
194 | bool seen_ax_reg; | 200 | bool seen_ax_reg; |
195 | }; | 201 | }; |
196 | 202 | ||
197 | /* maximum number of bytes emitted while JITing one eBPF insn */ | 203 | /* Maximum number of bytes emitted while JITing one eBPF insn */ |
198 | #define BPF_MAX_INSN_SIZE 128 | 204 | #define BPF_MAX_INSN_SIZE 128 |
199 | #define BPF_INSN_SAFETY 64 | 205 | #define BPF_INSN_SAFETY 64 |
200 | 206 | ||
201 | #define AUX_STACK_SPACE \ | 207 | #define AUX_STACK_SPACE \ |
202 | (32 /* space for rbx, r13, r14, r15 */ + \ | 208 | (32 /* Space for RBX, R13, R14, R15 */ + \ |
203 | 8 /* space for skb_copy_bits() buffer */) | 209 | 8 /* Space for skb_copy_bits() buffer */) |
204 | 210 | ||
205 | #define PROLOGUE_SIZE 37 | 211 | #define PROLOGUE_SIZE 37 |
206 | 212 | ||
207 | /* emit x64 prologue code for BPF program and check it's size. | 213 | /* |
214 | * Emit x86-64 prologue code for BPF program and check its size. | ||
208 | * bpf_tail_call helper will skip it while jumping into another program | 215 | * bpf_tail_call helper will skip it while jumping into another program |
209 | */ | 216 | */ |
210 | static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | 217 | static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) |
@@ -212,8 +219,11 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
212 | u8 *prog = *pprog; | 219 | u8 *prog = *pprog; |
213 | int cnt = 0; | 220 | int cnt = 0; |
214 | 221 | ||
215 | EMIT1(0x55); /* push rbp */ | 222 | /* push rbp */ |
216 | EMIT3(0x48, 0x89, 0xE5); /* mov rbp,rsp */ | 223 | EMIT1(0x55); |
224 | |||
225 | /* mov rbp,rsp */ | ||
226 | EMIT3(0x48, 0x89, 0xE5); | ||
217 | 227 | ||
218 | /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */ | 228 | /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */ |
219 | EMIT3_off32(0x48, 0x81, 0xEC, | 229 | EMIT3_off32(0x48, 0x81, 0xEC, |
@@ -222,14 +232,15 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
222 | /* sub rbp, AUX_STACK_SPACE */ | 232 | /* sub rbp, AUX_STACK_SPACE */ |
223 | EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE); | 233 | EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE); |
224 | 234 | ||
225 | /* all classic BPF filters use R6(rbx) save it */ | 235 | /* All classic BPF filters use R6(rbx) save it */ |
226 | 236 | ||
227 | /* mov qword ptr [rbp+0],rbx */ | 237 | /* mov qword ptr [rbp+0],rbx */ |
228 | EMIT4(0x48, 0x89, 0x5D, 0); | 238 | EMIT4(0x48, 0x89, 0x5D, 0); |
229 | 239 | ||
230 | /* bpf_convert_filter() maps classic BPF register X to R7 and uses R8 | 240 | /* |
231 | * as temporary, so all tcpdump filters need to spill/fill R7(r13) and | 241 | * bpf_convert_filter() maps classic BPF register X to R7 and uses R8 |
232 | * R8(r14). R9(r15) spill could be made conditional, but there is only | 242 | * as temporary, so all tcpdump filters need to spill/fill R7(R13) and |
243 | * R8(R14). R9(R15) spill could be made conditional, but there is only | ||
233 | * one 'bpf_error' return path out of helper functions inside bpf_jit.S | 244 | * one 'bpf_error' return path out of helper functions inside bpf_jit.S |
234 | * The overhead of extra spill is negligible for any filter other | 245 | * The overhead of extra spill is negligible for any filter other |
235 | * than synthetic ones. Therefore not worth adding complexity. | 246 | * than synthetic ones. Therefore not worth adding complexity. |
@@ -243,9 +254,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
243 | EMIT4(0x4C, 0x89, 0x7D, 24); | 254 | EMIT4(0x4C, 0x89, 0x7D, 24); |
244 | 255 | ||
245 | if (!ebpf_from_cbpf) { | 256 | if (!ebpf_from_cbpf) { |
246 | /* Clear the tail call counter (tail_call_cnt): for eBPF tail | 257 | /* |
258 | * Clear the tail call counter (tail_call_cnt): for eBPF tail | ||
247 | * calls we need to reset the counter to 0. It's done in two | 259 | * calls we need to reset the counter to 0. It's done in two |
248 | * instructions, resetting rax register to 0, and moving it | 260 | * instructions, resetting RAX register to 0, and moving it |
249 | * to the counter location. | 261 | * to the counter location. |
250 | */ | 262 | */ |
251 | 263 | ||
@@ -260,7 +272,9 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
260 | *pprog = prog; | 272 | *pprog = prog; |
261 | } | 273 | } |
262 | 274 | ||
263 | /* generate the following code: | 275 | /* |
276 | * Generate the following code: | ||
277 | * | ||
264 | * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... | 278 | * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... |
265 | * if (index >= array->map.max_entries) | 279 | * if (index >= array->map.max_entries) |
266 | * goto out; | 280 | * goto out; |
@@ -278,23 +292,26 @@ static void emit_bpf_tail_call(u8 **pprog) | |||
278 | int label1, label2, label3; | 292 | int label1, label2, label3; |
279 | int cnt = 0; | 293 | int cnt = 0; |
280 | 294 | ||
281 | /* rdi - pointer to ctx | 295 | /* |
296 | * rdi - pointer to ctx | ||
282 | * rsi - pointer to bpf_array | 297 | * rsi - pointer to bpf_array |
283 | * rdx - index in bpf_array | 298 | * rdx - index in bpf_array |
284 | */ | 299 | */ |
285 | 300 | ||
286 | /* if (index >= array->map.max_entries) | 301 | /* |
287 | * goto out; | 302 | * if (index >= array->map.max_entries) |
303 | * goto out; | ||
288 | */ | 304 | */ |
289 | EMIT2(0x89, 0xD2); /* mov edx, edx */ | 305 | EMIT2(0x89, 0xD2); /* mov edx, edx */ |
290 | EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ | 306 | EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ |
291 | offsetof(struct bpf_array, map.max_entries)); | 307 | offsetof(struct bpf_array, map.max_entries)); |
292 | #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* number of bytes to jump */ | 308 | #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */ |
293 | EMIT2(X86_JBE, OFFSET1); /* jbe out */ | 309 | EMIT2(X86_JBE, OFFSET1); /* jbe out */ |
294 | label1 = cnt; | 310 | label1 = cnt; |
295 | 311 | ||
296 | /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) | 312 | /* |
297 | * goto out; | 313 | * if (tail_call_cnt > MAX_TAIL_CALL_CNT) |
314 | * goto out; | ||
298 | */ | 315 | */ |
299 | EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */ | 316 | EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */ |
300 | EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ | 317 | EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ |
@@ -308,8 +325,9 @@ static void emit_bpf_tail_call(u8 **pprog) | |||
308 | EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ | 325 | EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ |
309 | offsetof(struct bpf_array, ptrs)); | 326 | offsetof(struct bpf_array, ptrs)); |
310 | 327 | ||
311 | /* if (prog == NULL) | 328 | /* |
312 | * goto out; | 329 | * if (prog == NULL) |
330 | * goto out; | ||
313 | */ | 331 | */ |
314 | EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ | 332 | EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ |
315 | #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE) | 333 | #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE) |
@@ -321,7 +339,8 @@ static void emit_bpf_tail_call(u8 **pprog) | |||
321 | offsetof(struct bpf_prog, bpf_func)); | 339 | offsetof(struct bpf_prog, bpf_func)); |
322 | EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ | 340 | EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ |
323 | 341 | ||
324 | /* now we're ready to jump into next BPF program | 342 | /* |
343 | * Wow we're ready to jump into next BPF program | ||
325 | * rdi == ctx (1st arg) | 344 | * rdi == ctx (1st arg) |
326 | * rax == prog->bpf_func + prologue_size | 345 | * rax == prog->bpf_func + prologue_size |
327 | */ | 346 | */ |
@@ -340,7 +359,8 @@ static void emit_load_skb_data_hlen(u8 **pprog) | |||
340 | u8 *prog = *pprog; | 359 | u8 *prog = *pprog; |
341 | int cnt = 0; | 360 | int cnt = 0; |
342 | 361 | ||
343 | /* r9d = skb->len - skb->data_len (headlen) | 362 | /* |
363 | * r9d = skb->len - skb->data_len (headlen) | ||
344 | * r10 = skb->data | 364 | * r10 = skb->data |
345 | */ | 365 | */ |
346 | /* mov %r9d, off32(%rdi) */ | 366 | /* mov %r9d, off32(%rdi) */ |
@@ -361,7 +381,8 @@ static void emit_mov_imm32(u8 **pprog, bool sign_propagate, | |||
361 | u8 b1, b2, b3; | 381 | u8 b1, b2, b3; |
362 | int cnt = 0; | 382 | int cnt = 0; |
363 | 383 | ||
364 | /* optimization: if imm32 is positive, use 'mov %eax, imm32' | 384 | /* |
385 | * Optimization: if imm32 is positive, use 'mov %eax, imm32' | ||
365 | * (which zero-extends imm32) to save 2 bytes. | 386 | * (which zero-extends imm32) to save 2 bytes. |
366 | */ | 387 | */ |
367 | if (sign_propagate && (s32)imm32 < 0) { | 388 | if (sign_propagate && (s32)imm32 < 0) { |
@@ -373,7 +394,8 @@ static void emit_mov_imm32(u8 **pprog, bool sign_propagate, | |||
373 | goto done; | 394 | goto done; |
374 | } | 395 | } |
375 | 396 | ||
376 | /* optimization: if imm32 is zero, use 'xor %eax, %eax' | 397 | /* |
398 | * Optimization: if imm32 is zero, use 'xor %eax, %eax' | ||
377 | * to save 3 bytes. | 399 | * to save 3 bytes. |
378 | */ | 400 | */ |
379 | if (imm32 == 0) { | 401 | if (imm32 == 0) { |
@@ -400,7 +422,8 @@ static void emit_mov_imm64(u8 **pprog, u32 dst_reg, | |||
400 | int cnt = 0; | 422 | int cnt = 0; |
401 | 423 | ||
402 | if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) { | 424 | if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) { |
403 | /* For emitting plain u32, where sign bit must not be | 425 | /* |
426 | * For emitting plain u32, where sign bit must not be | ||
404 | * propagated LLVM tends to load imm64 over mov32 | 427 | * propagated LLVM tends to load imm64 over mov32 |
405 | * directly, so save couple of bytes by just doing | 428 | * directly, so save couple of bytes by just doing |
406 | * 'mov %eax, imm32' instead. | 429 | * 'mov %eax, imm32' instead. |
@@ -525,7 +548,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
525 | else if (is_ereg(dst_reg)) | 548 | else if (is_ereg(dst_reg)) |
526 | EMIT1(add_1mod(0x40, dst_reg)); | 549 | EMIT1(add_1mod(0x40, dst_reg)); |
527 | 550 | ||
528 | /* b3 holds 'normal' opcode, b2 short form only valid | 551 | /* |
552 | * b3 holds 'normal' opcode, b2 short form only valid | ||
529 | * in case dst is eax/rax. | 553 | * in case dst is eax/rax. |
530 | */ | 554 | */ |
531 | switch (BPF_OP(insn->code)) { | 555 | switch (BPF_OP(insn->code)) { |
@@ -593,7 +617,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
593 | /* mov rax, dst_reg */ | 617 | /* mov rax, dst_reg */ |
594 | EMIT_mov(BPF_REG_0, dst_reg); | 618 | EMIT_mov(BPF_REG_0, dst_reg); |
595 | 619 | ||
596 | /* xor edx, edx | 620 | /* |
621 | * xor edx, edx | ||
597 | * equivalent to 'xor rdx, rdx', but one byte less | 622 | * equivalent to 'xor rdx, rdx', but one byte less |
598 | */ | 623 | */ |
599 | EMIT2(0x31, 0xd2); | 624 | EMIT2(0x31, 0xd2); |
@@ -655,7 +680,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
655 | } | 680 | } |
656 | break; | 681 | break; |
657 | } | 682 | } |
658 | /* shifts */ | 683 | /* Shifts */ |
659 | case BPF_ALU | BPF_LSH | BPF_K: | 684 | case BPF_ALU | BPF_LSH | BPF_K: |
660 | case BPF_ALU | BPF_RSH | BPF_K: | 685 | case BPF_ALU | BPF_RSH | BPF_K: |
661 | case BPF_ALU | BPF_ARSH | BPF_K: | 686 | case BPF_ALU | BPF_ARSH | BPF_K: |
@@ -686,7 +711,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
686 | case BPF_ALU64 | BPF_RSH | BPF_X: | 711 | case BPF_ALU64 | BPF_RSH | BPF_X: |
687 | case BPF_ALU64 | BPF_ARSH | BPF_X: | 712 | case BPF_ALU64 | BPF_ARSH | BPF_X: |
688 | 713 | ||
689 | /* check for bad case when dst_reg == rcx */ | 714 | /* Check for bad case when dst_reg == rcx */ |
690 | if (dst_reg == BPF_REG_4) { | 715 | if (dst_reg == BPF_REG_4) { |
691 | /* mov r11, dst_reg */ | 716 | /* mov r11, dst_reg */ |
692 | EMIT_mov(AUX_REG, dst_reg); | 717 | EMIT_mov(AUX_REG, dst_reg); |
@@ -724,13 +749,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
724 | case BPF_ALU | BPF_END | BPF_FROM_BE: | 749 | case BPF_ALU | BPF_END | BPF_FROM_BE: |
725 | switch (imm32) { | 750 | switch (imm32) { |
726 | case 16: | 751 | case 16: |
727 | /* emit 'ror %ax, 8' to swap lower 2 bytes */ | 752 | /* Emit 'ror %ax, 8' to swap lower 2 bytes */ |
728 | EMIT1(0x66); | 753 | EMIT1(0x66); |
729 | if (is_ereg(dst_reg)) | 754 | if (is_ereg(dst_reg)) |
730 | EMIT1(0x41); | 755 | EMIT1(0x41); |
731 | EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); | 756 | EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); |
732 | 757 | ||
733 | /* emit 'movzwl eax, ax' */ | 758 | /* Emit 'movzwl eax, ax' */ |
734 | if (is_ereg(dst_reg)) | 759 | if (is_ereg(dst_reg)) |
735 | EMIT3(0x45, 0x0F, 0xB7); | 760 | EMIT3(0x45, 0x0F, 0xB7); |
736 | else | 761 | else |
@@ -738,7 +763,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
738 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); | 763 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); |
739 | break; | 764 | break; |
740 | case 32: | 765 | case 32: |
741 | /* emit 'bswap eax' to swap lower 4 bytes */ | 766 | /* Emit 'bswap eax' to swap lower 4 bytes */ |
742 | if (is_ereg(dst_reg)) | 767 | if (is_ereg(dst_reg)) |
743 | EMIT2(0x41, 0x0F); | 768 | EMIT2(0x41, 0x0F); |
744 | else | 769 | else |
@@ -746,7 +771,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
746 | EMIT1(add_1reg(0xC8, dst_reg)); | 771 | EMIT1(add_1reg(0xC8, dst_reg)); |
747 | break; | 772 | break; |
748 | case 64: | 773 | case 64: |
749 | /* emit 'bswap rax' to swap 8 bytes */ | 774 | /* Emit 'bswap rax' to swap 8 bytes */ |
750 | EMIT3(add_1mod(0x48, dst_reg), 0x0F, | 775 | EMIT3(add_1mod(0x48, dst_reg), 0x0F, |
751 | add_1reg(0xC8, dst_reg)); | 776 | add_1reg(0xC8, dst_reg)); |
752 | break; | 777 | break; |
@@ -756,7 +781,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
756 | case BPF_ALU | BPF_END | BPF_FROM_LE: | 781 | case BPF_ALU | BPF_END | BPF_FROM_LE: |
757 | switch (imm32) { | 782 | switch (imm32) { |
758 | case 16: | 783 | case 16: |
759 | /* emit 'movzwl eax, ax' to zero extend 16-bit | 784 | /* |
785 | * Emit 'movzwl eax, ax' to zero extend 16-bit | ||
760 | * into 64 bit | 786 | * into 64 bit |
761 | */ | 787 | */ |
762 | if (is_ereg(dst_reg)) | 788 | if (is_ereg(dst_reg)) |
@@ -766,7 +792,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
766 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); | 792 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); |
767 | break; | 793 | break; |
768 | case 32: | 794 | case 32: |
769 | /* emit 'mov eax, eax' to clear upper 32-bits */ | 795 | /* Emit 'mov eax, eax' to clear upper 32-bits */ |
770 | if (is_ereg(dst_reg)) | 796 | if (is_ereg(dst_reg)) |
771 | EMIT1(0x45); | 797 | EMIT1(0x45); |
772 | EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); | 798 | EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); |
@@ -809,9 +835,9 @@ st: if (is_imm8(insn->off)) | |||
809 | 835 | ||
810 | /* STX: *(u8*)(dst_reg + off) = src_reg */ | 836 | /* STX: *(u8*)(dst_reg + off) = src_reg */ |
811 | case BPF_STX | BPF_MEM | BPF_B: | 837 | case BPF_STX | BPF_MEM | BPF_B: |
812 | /* emit 'mov byte ptr [rax + off], al' */ | 838 | /* Emit 'mov byte ptr [rax + off], al' */ |
813 | if (is_ereg(dst_reg) || is_ereg(src_reg) || | 839 | if (is_ereg(dst_reg) || is_ereg(src_reg) || |
814 | /* have to add extra byte for x86 SIL, DIL regs */ | 840 | /* We have to add extra byte for x86 SIL, DIL regs */ |
815 | src_reg == BPF_REG_1 || src_reg == BPF_REG_2) | 841 | src_reg == BPF_REG_1 || src_reg == BPF_REG_2) |
816 | EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); | 842 | EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); |
817 | else | 843 | else |
@@ -840,25 +866,26 @@ stx: if (is_imm8(insn->off)) | |||
840 | 866 | ||
841 | /* LDX: dst_reg = *(u8*)(src_reg + off) */ | 867 | /* LDX: dst_reg = *(u8*)(src_reg + off) */ |
842 | case BPF_LDX | BPF_MEM | BPF_B: | 868 | case BPF_LDX | BPF_MEM | BPF_B: |
843 | /* emit 'movzx rax, byte ptr [rax + off]' */ | 869 | /* Emit 'movzx rax, byte ptr [rax + off]' */ |
844 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); | 870 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); |
845 | goto ldx; | 871 | goto ldx; |
846 | case BPF_LDX | BPF_MEM | BPF_H: | 872 | case BPF_LDX | BPF_MEM | BPF_H: |
847 | /* emit 'movzx rax, word ptr [rax + off]' */ | 873 | /* Emit 'movzx rax, word ptr [rax + off]' */ |
848 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); | 874 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); |
849 | goto ldx; | 875 | goto ldx; |
850 | case BPF_LDX | BPF_MEM | BPF_W: | 876 | case BPF_LDX | BPF_MEM | BPF_W: |
851 | /* emit 'mov eax, dword ptr [rax+0x14]' */ | 877 | /* Emit 'mov eax, dword ptr [rax+0x14]' */ |
852 | if (is_ereg(dst_reg) || is_ereg(src_reg)) | 878 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
853 | EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); | 879 | EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); |
854 | else | 880 | else |
855 | EMIT1(0x8B); | 881 | EMIT1(0x8B); |
856 | goto ldx; | 882 | goto ldx; |
857 | case BPF_LDX | BPF_MEM | BPF_DW: | 883 | case BPF_LDX | BPF_MEM | BPF_DW: |
858 | /* emit 'mov rax, qword ptr [rax+0x14]' */ | 884 | /* Emit 'mov rax, qword ptr [rax+0x14]' */ |
859 | EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); | 885 | EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); |
860 | ldx: /* if insn->off == 0 we can save one extra byte, but | 886 | ldx: /* |
861 | * special case of x86 r13 which always needs an offset | 887 | * If insn->off == 0 we can save one extra byte, but |
888 | * special case of x86 R13 which always needs an offset | ||
862 | * is not worth the hassle | 889 | * is not worth the hassle |
863 | */ | 890 | */ |
864 | if (is_imm8(insn->off)) | 891 | if (is_imm8(insn->off)) |
@@ -870,7 +897,7 @@ ldx: /* if insn->off == 0 we can save one extra byte, but | |||
870 | 897 | ||
871 | /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */ | 898 | /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */ |
872 | case BPF_STX | BPF_XADD | BPF_W: | 899 | case BPF_STX | BPF_XADD | BPF_W: |
873 | /* emit 'lock add dword ptr [rax + off], eax' */ | 900 | /* Emit 'lock add dword ptr [rax + off], eax' */ |
874 | if (is_ereg(dst_reg) || is_ereg(src_reg)) | 901 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
875 | EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); | 902 | EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); |
876 | else | 903 | else |
@@ -897,14 +924,15 @@ xadd: if (is_imm8(insn->off)) | |||
897 | } else { | 924 | } else { |
898 | EMIT2(0x41, 0x52); /* push %r10 */ | 925 | EMIT2(0x41, 0x52); /* push %r10 */ |
899 | EMIT2(0x41, 0x51); /* push %r9 */ | 926 | EMIT2(0x41, 0x51); /* push %r9 */ |
900 | /* need to adjust jmp offset, since | 927 | /* |
928 | * We need to adjust jmp offset, since | ||
901 | * pop %r9, pop %r10 take 4 bytes after call insn | 929 | * pop %r9, pop %r10 take 4 bytes after call insn |
902 | */ | 930 | */ |
903 | jmp_offset += 4; | 931 | jmp_offset += 4; |
904 | } | 932 | } |
905 | } | 933 | } |
906 | if (!imm32 || !is_simm32(jmp_offset)) { | 934 | if (!imm32 || !is_simm32(jmp_offset)) { |
907 | pr_err("unsupported bpf func %d addr %p image %p\n", | 935 | pr_err("unsupported BPF func %d addr %p image %p\n", |
908 | imm32, func, image); | 936 | imm32, func, image); |
909 | return -EINVAL; | 937 | return -EINVAL; |
910 | } | 938 | } |
@@ -970,7 +998,7 @@ xadd: if (is_imm8(insn->off)) | |||
970 | else | 998 | else |
971 | EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); | 999 | EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); |
972 | 1000 | ||
973 | emit_cond_jmp: /* convert BPF opcode to x86 */ | 1001 | emit_cond_jmp: /* Convert BPF opcode to x86 */ |
974 | switch (BPF_OP(insn->code)) { | 1002 | switch (BPF_OP(insn->code)) { |
975 | case BPF_JEQ: | 1003 | case BPF_JEQ: |
976 | jmp_cond = X86_JE; | 1004 | jmp_cond = X86_JE; |
@@ -996,22 +1024,22 @@ emit_cond_jmp: /* convert BPF opcode to x86 */ | |||
996 | jmp_cond = X86_JBE; | 1024 | jmp_cond = X86_JBE; |
997 | break; | 1025 | break; |
998 | case BPF_JSGT: | 1026 | case BPF_JSGT: |
999 | /* signed '>', GT in x86 */ | 1027 | /* Signed '>', GT in x86 */ |
1000 | jmp_cond = X86_JG; | 1028 | jmp_cond = X86_JG; |
1001 | break; | 1029 | break; |
1002 | case BPF_JSLT: | 1030 | case BPF_JSLT: |
1003 | /* signed '<', LT in x86 */ | 1031 | /* Signed '<', LT in x86 */ |
1004 | jmp_cond = X86_JL; | 1032 | jmp_cond = X86_JL; |
1005 | break; | 1033 | break; |
1006 | case BPF_JSGE: | 1034 | case BPF_JSGE: |
1007 | /* signed '>=', GE in x86 */ | 1035 | /* Signed '>=', GE in x86 */ |
1008 | jmp_cond = X86_JGE; | 1036 | jmp_cond = X86_JGE; |
1009 | break; | 1037 | break; |
1010 | case BPF_JSLE: | 1038 | case BPF_JSLE: |
1011 | /* signed '<=', LE in x86 */ | 1039 | /* Signed '<=', LE in x86 */ |
1012 | jmp_cond = X86_JLE; | 1040 | jmp_cond = X86_JLE; |
1013 | break; | 1041 | break; |
1014 | default: /* to silence gcc warning */ | 1042 | default: /* to silence GCC warning */ |
1015 | return -EFAULT; | 1043 | return -EFAULT; |
1016 | } | 1044 | } |
1017 | jmp_offset = addrs[i + insn->off] - addrs[i]; | 1045 | jmp_offset = addrs[i + insn->off] - addrs[i]; |
@@ -1039,7 +1067,7 @@ emit_cond_jmp: /* convert BPF opcode to x86 */ | |||
1039 | jmp_offset = addrs[i + insn->off] - addrs[i]; | 1067 | jmp_offset = addrs[i + insn->off] - addrs[i]; |
1040 | 1068 | ||
1041 | if (!jmp_offset) | 1069 | if (!jmp_offset) |
1042 | /* optimize out nop jumps */ | 1070 | /* Optimize out nop jumps */ |
1043 | break; | 1071 | break; |
1044 | emit_jmp: | 1072 | emit_jmp: |
1045 | if (is_imm8(jmp_offset)) { | 1073 | if (is_imm8(jmp_offset)) { |
@@ -1061,7 +1089,7 @@ common_load: | |||
1061 | ctx->seen_ld_abs = seen_ld_abs = true; | 1089 | ctx->seen_ld_abs = seen_ld_abs = true; |
1062 | jmp_offset = func - (image + addrs[i]); | 1090 | jmp_offset = func - (image + addrs[i]); |
1063 | if (!func || !is_simm32(jmp_offset)) { | 1091 | if (!func || !is_simm32(jmp_offset)) { |
1064 | pr_err("unsupported bpf func %d addr %p image %p\n", | 1092 | pr_err("unsupported BPF func %d addr %p image %p\n", |
1065 | imm32, func, image); | 1093 | imm32, func, image); |
1066 | return -EINVAL; | 1094 | return -EINVAL; |
1067 | } | 1095 | } |
@@ -1080,7 +1108,8 @@ common_load: | |||
1080 | EMIT2_off32(0x81, 0xC6, imm32); | 1108 | EMIT2_off32(0x81, 0xC6, imm32); |
1081 | } | 1109 | } |
1082 | } | 1110 | } |
1083 | /* skb pointer is in R6 (%rbx), it will be copied into | 1111 | /* |
1112 | * skb pointer is in R6 (%rbx), it will be copied into | ||
1084 | * %rdi if skb_copy_bits() call is necessary. | 1113 | * %rdi if skb_copy_bits() call is necessary. |
1085 | * sk_load_* helpers also use %r10 and %r9d. | 1114 | * sk_load_* helpers also use %r10 and %r9d. |
1086 | * See bpf_jit.S | 1115 | * See bpf_jit.S |
@@ -1111,7 +1140,7 @@ common_load: | |||
1111 | goto emit_jmp; | 1140 | goto emit_jmp; |
1112 | } | 1141 | } |
1113 | seen_exit = true; | 1142 | seen_exit = true; |
1114 | /* update cleanup_addr */ | 1143 | /* Update cleanup_addr */ |
1115 | ctx->cleanup_addr = proglen; | 1144 | ctx->cleanup_addr = proglen; |
1116 | /* mov rbx, qword ptr [rbp+0] */ | 1145 | /* mov rbx, qword ptr [rbp+0] */ |
1117 | EMIT4(0x48, 0x8B, 0x5D, 0); | 1146 | EMIT4(0x48, 0x8B, 0x5D, 0); |
@@ -1129,10 +1158,11 @@ common_load: | |||
1129 | break; | 1158 | break; |
1130 | 1159 | ||
1131 | default: | 1160 | default: |
1132 | /* By design x64 JIT should support all BPF instructions | 1161 | /* |
1162 | * By design x86-64 JIT should support all BPF instructions. | ||
1133 | * This error will be seen if new instruction was added | 1163 | * This error will be seen if new instruction was added |
1134 | * to interpreter, but not to JIT | 1164 | * to the interpreter, but not to the JIT, or if there is |
1135 | * or if there is junk in bpf_prog | 1165 | * junk in bpf_prog. |
1136 | */ | 1166 | */ |
1137 | pr_err("bpf_jit: unknown opcode %02x\n", insn->code); | 1167 | pr_err("bpf_jit: unknown opcode %02x\n", insn->code); |
1138 | return -EINVAL; | 1168 | return -EINVAL; |
@@ -1184,7 +1214,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) | |||
1184 | return orig_prog; | 1214 | return orig_prog; |
1185 | 1215 | ||
1186 | tmp = bpf_jit_blind_constants(prog); | 1216 | tmp = bpf_jit_blind_constants(prog); |
1187 | /* If blinding was requested and we failed during blinding, | 1217 | /* |
1218 | * If blinding was requested and we failed during blinding, | ||
1188 | * we must fall back to the interpreter. | 1219 | * we must fall back to the interpreter. |
1189 | */ | 1220 | */ |
1190 | if (IS_ERR(tmp)) | 1221 | if (IS_ERR(tmp)) |
@@ -1218,8 +1249,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) | |||
1218 | goto out_addrs; | 1249 | goto out_addrs; |
1219 | } | 1250 | } |
1220 | 1251 | ||
1221 | /* Before first pass, make a rough estimation of addrs[] | 1252 | /* |
1222 | * each bpf instruction is translated to less than 64 bytes | 1253 | * Before first pass, make a rough estimation of addrs[] |
1254 | * each BPF instruction is translated to less than 64 bytes | ||
1223 | */ | 1255 | */ |
1224 | for (proglen = 0, i = 0; i < prog->len; i++) { | 1256 | for (proglen = 0, i = 0; i < prog->len; i++) { |
1225 | proglen += 64; | 1257 | proglen += 64; |
@@ -1228,10 +1260,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) | |||
1228 | ctx.cleanup_addr = proglen; | 1260 | ctx.cleanup_addr = proglen; |
1229 | skip_init_addrs: | 1261 | skip_init_addrs: |
1230 | 1262 | ||
1231 | /* JITed image shrinks with every pass and the loop iterates | 1263 | /* |
1232 | * until the image stops shrinking. Very large bpf programs | 1264 | * JITed image shrinks with every pass and the loop iterates |
1265 | * until the image stops shrinking. Very large BPF programs | ||
1233 | * may converge on the last pass. In such case do one more | 1266 | * may converge on the last pass. In such case do one more |
1234 | * pass to emit the final image | 1267 | * pass to emit the final image. |
1235 | */ | 1268 | */ |
1236 | for (pass = 0; pass < 20 || image; pass++) { | 1269 | for (pass = 0; pass < 20 || image; pass++) { |
1237 | proglen = do_jit(prog, addrs, image, oldproglen, &ctx); | 1270 | proglen = do_jit(prog, addrs, image, oldproglen, &ctx); |