diff options
| -rw-r--r-- | arch/x86/events/amd/ibs.c | 2 | ||||
| -rw-r--r-- | arch/x86/kernel/apm_32.c | 2 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/mtrr/Makefile | 2 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/mtrr/mtrr.c (renamed from arch/x86/kernel/cpu/mtrr/main.c) | 33 | ||||
| -rw-r--r-- | arch/x86/kernel/e820.c | 32 | ||||
| -rw-r--r-- | arch/x86/kernel/early-quirks.c | 8 | ||||
| -rw-r--r-- | arch/x86/kernel/hpet.c | 5 | ||||
| -rw-r--r-- | arch/x86/kernel/uprobes.c | 4 | ||||
| -rw-r--r-- | arch/x86/mm/numa.c | 22 | ||||
| -rw-r--r-- | arch/x86/net/bpf_jit_comp.c | 233 |
10 files changed, 188 insertions, 155 deletions
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 786fd875de92..4b98101209a1 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c | |||
| @@ -889,7 +889,7 @@ static void force_ibs_eilvt_setup(void) | |||
| 889 | if (!ibs_eilvt_valid()) | 889 | if (!ibs_eilvt_valid()) |
| 890 | goto out; | 890 | goto out; |
| 891 | 891 | ||
| 892 | pr_info("IBS: LVT offset %d assigned\n", offset); | 892 | pr_info("LVT offset %d assigned\n", offset); |
| 893 | 893 | ||
| 894 | return; | 894 | return; |
| 895 | out: | 895 | out: |
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index cadeafabf167..5d0de79fdab0 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c | |||
| @@ -2433,7 +2433,7 @@ MODULE_PARM_DESC(idle_threshold, | |||
| 2433 | "System idle percentage above which to make APM BIOS idle calls"); | 2433 | "System idle percentage above which to make APM BIOS idle calls"); |
| 2434 | module_param(idle_period, int, 0444); | 2434 | module_param(idle_period, int, 0444); |
| 2435 | MODULE_PARM_DESC(idle_period, | 2435 | MODULE_PARM_DESC(idle_period, |
| 2436 | "Period (in sec/100) over which to caculate the idle percentage"); | 2436 | "Period (in sec/100) over which to calculate the idle percentage"); |
| 2437 | module_param(smp, bool, 0444); | 2437 | module_param(smp, bool, 0444); |
| 2438 | MODULE_PARM_DESC(smp, | 2438 | MODULE_PARM_DESC(smp, |
| 2439 | "Set this to enable APM use on an SMP platform. Use with caution on older systems"); | 2439 | "Set this to enable APM use on an SMP platform. Use with caution on older systems"); |
diff --git a/arch/x86/kernel/cpu/mtrr/Makefile b/arch/x86/kernel/cpu/mtrr/Makefile index ad9e5ed81181..2ad9107ee980 100644 --- a/arch/x86/kernel/cpu/mtrr/Makefile +++ b/arch/x86/kernel/cpu/mtrr/Makefile | |||
| @@ -1,3 +1,3 @@ | |||
| 1 | obj-y := main.o if.o generic.o cleanup.o | 1 | obj-y := mtrr.o if.o generic.o cleanup.o |
| 2 | obj-$(CONFIG_X86_32) += amd.o cyrix.o centaur.o | 2 | obj-$(CONFIG_X86_32) += amd.o cyrix.o centaur.o |
| 3 | 3 | ||
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/mtrr.c index 3ea0047beb40..9a19c800fe40 100644 --- a/arch/x86/kernel/cpu/mtrr/main.c +++ b/arch/x86/kernel/cpu/mtrr/mtrr.c | |||
| @@ -101,7 +101,7 @@ static int have_wrcomb(void) | |||
| 101 | if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS && | 101 | if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS && |
| 102 | dev->device == PCI_DEVICE_ID_SERVERWORKS_LE && | 102 | dev->device == PCI_DEVICE_ID_SERVERWORKS_LE && |
| 103 | dev->revision <= 5) { | 103 | dev->revision <= 5) { |
| 104 | pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n"); | 104 | pr_info("Serverworks LE rev < 6 detected. Write-combining disabled.\n"); |
| 105 | pci_dev_put(dev); | 105 | pci_dev_put(dev); |
| 106 | return 0; | 106 | return 0; |
| 107 | } | 107 | } |
| @@ -111,7 +111,7 @@ static int have_wrcomb(void) | |||
| 111 | */ | 111 | */ |
| 112 | if (dev->vendor == PCI_VENDOR_ID_INTEL && | 112 | if (dev->vendor == PCI_VENDOR_ID_INTEL && |
| 113 | dev->device == PCI_DEVICE_ID_INTEL_82451NX) { | 113 | dev->device == PCI_DEVICE_ID_INTEL_82451NX) { |
| 114 | pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n"); | 114 | pr_info("Intel 450NX MMC detected. Write-combining disabled.\n"); |
| 115 | pci_dev_put(dev); | 115 | pci_dev_put(dev); |
| 116 | return 0; | 116 | return 0; |
| 117 | } | 117 | } |
| @@ -313,24 +313,24 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
| 313 | return error; | 313 | return error; |
| 314 | 314 | ||
| 315 | if (type >= MTRR_NUM_TYPES) { | 315 | if (type >= MTRR_NUM_TYPES) { |
| 316 | pr_warn("mtrr: type: %u invalid\n", type); | 316 | pr_warn("type: %u invalid\n", type); |
| 317 | return -EINVAL; | 317 | return -EINVAL; |
| 318 | } | 318 | } |
| 319 | 319 | ||
| 320 | /* If the type is WC, check that this processor supports it */ | 320 | /* If the type is WC, check that this processor supports it */ |
| 321 | if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) { | 321 | if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) { |
| 322 | pr_warn("mtrr: your processor doesn't support write-combining\n"); | 322 | pr_warn("your processor doesn't support write-combining\n"); |
| 323 | return -ENOSYS; | 323 | return -ENOSYS; |
| 324 | } | 324 | } |
| 325 | 325 | ||
| 326 | if (!size) { | 326 | if (!size) { |
| 327 | pr_warn("mtrr: zero sized request\n"); | 327 | pr_warn("zero sized request\n"); |
| 328 | return -EINVAL; | 328 | return -EINVAL; |
| 329 | } | 329 | } |
| 330 | 330 | ||
| 331 | if ((base | (base + size - 1)) >> | 331 | if ((base | (base + size - 1)) >> |
| 332 | (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) { | 332 | (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) { |
| 333 | pr_warn("mtrr: base or size exceeds the MTRR width\n"); | 333 | pr_warn("base or size exceeds the MTRR width\n"); |
| 334 | return -EINVAL; | 334 | return -EINVAL; |
| 335 | } | 335 | } |
| 336 | 336 | ||
| @@ -361,8 +361,7 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
| 361 | } else if (types_compatible(type, ltype)) | 361 | } else if (types_compatible(type, ltype)) |
| 362 | continue; | 362 | continue; |
| 363 | } | 363 | } |
| 364 | pr_warn("mtrr: 0x%lx000,0x%lx000 overlaps existing" | 364 | pr_warn("0x%lx000,0x%lx000 overlaps existing 0x%lx000,0x%lx000\n", base, size, lbase, |
| 365 | " 0x%lx000,0x%lx000\n", base, size, lbase, | ||
| 366 | lsize); | 365 | lsize); |
| 367 | goto out; | 366 | goto out; |
| 368 | } | 367 | } |
| @@ -370,7 +369,7 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
| 370 | if (ltype != type) { | 369 | if (ltype != type) { |
| 371 | if (types_compatible(type, ltype)) | 370 | if (types_compatible(type, ltype)) |
| 372 | continue; | 371 | continue; |
| 373 | pr_warn("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n", | 372 | pr_warn("type mismatch for %lx000,%lx000 old: %s new: %s\n", |
| 374 | base, size, mtrr_attrib_to_str(ltype), | 373 | base, size, mtrr_attrib_to_str(ltype), |
| 375 | mtrr_attrib_to_str(type)); | 374 | mtrr_attrib_to_str(type)); |
| 376 | goto out; | 375 | goto out; |
| @@ -396,7 +395,7 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
| 396 | } | 395 | } |
| 397 | } | 396 | } |
| 398 | } else { | 397 | } else { |
| 399 | pr_info("mtrr: no more MTRRs available\n"); | 398 | pr_info("no more MTRRs available\n"); |
| 400 | } | 399 | } |
| 401 | error = i; | 400 | error = i; |
| 402 | out: | 401 | out: |
| @@ -408,8 +407,8 @@ int mtrr_add_page(unsigned long base, unsigned long size, | |||
| 408 | static int mtrr_check(unsigned long base, unsigned long size) | 407 | static int mtrr_check(unsigned long base, unsigned long size) |
| 409 | { | 408 | { |
| 410 | if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) { | 409 | if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) { |
| 411 | pr_warn("mtrr: size and base must be multiples of 4 kiB\n"); | 410 | pr_warn("size and base must be multiples of 4 kiB\n"); |
| 412 | pr_debug("mtrr: size: 0x%lx base: 0x%lx\n", size, base); | 411 | pr_debug("size: 0x%lx base: 0x%lx\n", size, base); |
| 413 | dump_stack(); | 412 | dump_stack(); |
| 414 | return -1; | 413 | return -1; |
| 415 | } | 414 | } |
| @@ -500,22 +499,22 @@ int mtrr_del_page(int reg, unsigned long base, unsigned long size) | |||
| 500 | } | 499 | } |
| 501 | } | 500 | } |
| 502 | if (reg < 0) { | 501 | if (reg < 0) { |
| 503 | pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n", | 502 | pr_debug("no MTRR for %lx000,%lx000 found\n", |
| 504 | base, size); | 503 | base, size); |
| 505 | goto out; | 504 | goto out; |
| 506 | } | 505 | } |
| 507 | } | 506 | } |
| 508 | if (reg >= max) { | 507 | if (reg >= max) { |
| 509 | pr_warn("mtrr: register: %d too big\n", reg); | 508 | pr_warn("register: %d too big\n", reg); |
| 510 | goto out; | 509 | goto out; |
| 511 | } | 510 | } |
| 512 | mtrr_if->get(reg, &lbase, &lsize, <ype); | 511 | mtrr_if->get(reg, &lbase, &lsize, <ype); |
| 513 | if (lsize < 1) { | 512 | if (lsize < 1) { |
| 514 | pr_warn("mtrr: MTRR %d not used\n", reg); | 513 | pr_warn("MTRR %d not used\n", reg); |
| 515 | goto out; | 514 | goto out; |
| 516 | } | 515 | } |
| 517 | if (mtrr_usage_table[reg] < 1) { | 516 | if (mtrr_usage_table[reg] < 1) { |
| 518 | pr_warn("mtrr: reg: %d has count=0\n", reg); | 517 | pr_warn("reg: %d has count=0\n", reg); |
| 519 | goto out; | 518 | goto out; |
| 520 | } | 519 | } |
| 521 | if (--mtrr_usage_table[reg] < 1) | 520 | if (--mtrr_usage_table[reg] < 1) |
| @@ -776,7 +775,7 @@ void __init mtrr_bp_init(void) | |||
| 776 | } | 775 | } |
| 777 | 776 | ||
| 778 | if (!mtrr_enabled()) { | 777 | if (!mtrr_enabled()) { |
| 779 | pr_info("MTRR: Disabled\n"); | 778 | pr_info("Disabled\n"); |
| 780 | 779 | ||
| 781 | /* | 780 | /* |
| 782 | * PAT initialization relies on MTRR's rendezvous handler. | 781 | * PAT initialization relies on MTRR's rendezvous handler. |
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 6a2cb1442e05..d1f25c831447 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c | |||
| @@ -155,7 +155,8 @@ static void __init __e820__range_add(struct e820_table *table, u64 start, u64 si | |||
| 155 | int x = table->nr_entries; | 155 | int x = table->nr_entries; |
| 156 | 156 | ||
| 157 | if (x >= ARRAY_SIZE(table->entries)) { | 157 | if (x >= ARRAY_SIZE(table->entries)) { |
| 158 | pr_err("e820: too many entries; ignoring [mem %#010llx-%#010llx]\n", start, start + size - 1); | 158 | pr_err("too many entries; ignoring [mem %#010llx-%#010llx]\n", |
| 159 | start, start + size - 1); | ||
| 159 | return; | 160 | return; |
| 160 | } | 161 | } |
| 161 | 162 | ||
| @@ -190,9 +191,10 @@ void __init e820__print_table(char *who) | |||
| 190 | int i; | 191 | int i; |
| 191 | 192 | ||
| 192 | for (i = 0; i < e820_table->nr_entries; i++) { | 193 | for (i = 0; i < e820_table->nr_entries; i++) { |
| 193 | pr_info("%s: [mem %#018Lx-%#018Lx] ", who, | 194 | pr_info("%s: [mem %#018Lx-%#018Lx] ", |
| 194 | e820_table->entries[i].addr, | 195 | who, |
| 195 | e820_table->entries[i].addr + e820_table->entries[i].size - 1); | 196 | e820_table->entries[i].addr, |
| 197 | e820_table->entries[i].addr + e820_table->entries[i].size - 1); | ||
| 196 | 198 | ||
| 197 | e820_print_type(e820_table->entries[i].type); | 199 | e820_print_type(e820_table->entries[i].type); |
| 198 | pr_cont("\n"); | 200 | pr_cont("\n"); |
| @@ -574,7 +576,7 @@ void __init e820__update_table_print(void) | |||
| 574 | if (e820__update_table(e820_table)) | 576 | if (e820__update_table(e820_table)) |
| 575 | return; | 577 | return; |
| 576 | 578 | ||
| 577 | pr_info("e820: modified physical RAM map:\n"); | 579 | pr_info("modified physical RAM map:\n"); |
| 578 | e820__print_table("modified"); | 580 | e820__print_table("modified"); |
| 579 | } | 581 | } |
| 580 | 582 | ||
| @@ -636,9 +638,8 @@ __init void e820__setup_pci_gap(void) | |||
| 636 | if (!found) { | 638 | if (!found) { |
| 637 | #ifdef CONFIG_X86_64 | 639 | #ifdef CONFIG_X86_64 |
| 638 | gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; | 640 | gapstart = (max_pfn << PAGE_SHIFT) + 1024*1024; |
| 639 | pr_err( | 641 | pr_err("Cannot find an available gap in the 32-bit address range\n"); |
| 640 | "e820: Cannot find an available gap in the 32-bit address range\n" | 642 | pr_err("PCI devices with unassigned 32-bit BARs may not work!\n"); |
| 641 | "e820: PCI devices with unassigned 32-bit BARs may not work!\n"); | ||
| 642 | #else | 643 | #else |
| 643 | gapstart = 0x10000000; | 644 | gapstart = 0x10000000; |
| 644 | #endif | 645 | #endif |
| @@ -649,7 +650,8 @@ __init void e820__setup_pci_gap(void) | |||
| 649 | */ | 650 | */ |
| 650 | pci_mem_start = gapstart; | 651 | pci_mem_start = gapstart; |
| 651 | 652 | ||
| 652 | pr_info("e820: [mem %#010lx-%#010lx] available for PCI devices\n", gapstart, gapstart + gapsize - 1); | 653 | pr_info("[mem %#010lx-%#010lx] available for PCI devices\n", |
| 654 | gapstart, gapstart + gapsize - 1); | ||
| 653 | } | 655 | } |
| 654 | 656 | ||
| 655 | /* | 657 | /* |
| @@ -711,7 +713,7 @@ void __init e820__memory_setup_extended(u64 phys_addr, u32 data_len) | |||
| 711 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); | 713 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
| 712 | 714 | ||
| 713 | early_memunmap(sdata, data_len); | 715 | early_memunmap(sdata, data_len); |
| 714 | pr_info("e820: extended physical RAM map:\n"); | 716 | pr_info("extended physical RAM map:\n"); |
| 715 | e820__print_table("extended"); | 717 | e820__print_table("extended"); |
| 716 | } | 718 | } |
| 717 | 719 | ||
| @@ -780,7 +782,7 @@ u64 __init e820__memblock_alloc_reserved(u64 size, u64 align) | |||
| 780 | addr = __memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); | 782 | addr = __memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE); |
| 781 | if (addr) { | 783 | if (addr) { |
| 782 | e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); | 784 | e820__range_update_kexec(addr, size, E820_TYPE_RAM, E820_TYPE_RESERVED); |
| 783 | pr_info("e820: update e820_table_kexec for e820__memblock_alloc_reserved()\n"); | 785 | pr_info("update e820_table_kexec for e820__memblock_alloc_reserved()\n"); |
| 784 | e820__update_table_kexec(); | 786 | e820__update_table_kexec(); |
| 785 | } | 787 | } |
| 786 | 788 | ||
| @@ -830,8 +832,8 @@ static unsigned long __init e820_end_pfn(unsigned long limit_pfn, enum e820_type | |||
| 830 | if (last_pfn > max_arch_pfn) | 832 | if (last_pfn > max_arch_pfn) |
| 831 | last_pfn = max_arch_pfn; | 833 | last_pfn = max_arch_pfn; |
| 832 | 834 | ||
| 833 | pr_info("e820: last_pfn = %#lx max_arch_pfn = %#lx\n", | 835 | pr_info("last_pfn = %#lx max_arch_pfn = %#lx\n", |
| 834 | last_pfn, max_arch_pfn); | 836 | last_pfn, max_arch_pfn); |
| 835 | return last_pfn; | 837 | return last_pfn; |
| 836 | } | 838 | } |
| 837 | 839 | ||
| @@ -1005,7 +1007,7 @@ void __init e820__finish_early_params(void) | |||
| 1005 | if (e820__update_table(e820_table) < 0) | 1007 | if (e820__update_table(e820_table) < 0) |
| 1006 | early_panic("Invalid user supplied memory map"); | 1008 | early_panic("Invalid user supplied memory map"); |
| 1007 | 1009 | ||
| 1008 | pr_info("e820: user-defined physical RAM map:\n"); | 1010 | pr_info("user-defined physical RAM map:\n"); |
| 1009 | e820__print_table("user"); | 1011 | e820__print_table("user"); |
| 1010 | } | 1012 | } |
| 1011 | } | 1013 | } |
| @@ -1238,7 +1240,7 @@ void __init e820__memory_setup(void) | |||
| 1238 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); | 1240 | memcpy(e820_table_kexec, e820_table, sizeof(*e820_table_kexec)); |
| 1239 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); | 1241 | memcpy(e820_table_firmware, e820_table, sizeof(*e820_table_firmware)); |
| 1240 | 1242 | ||
| 1241 | pr_info("e820: BIOS-provided physical RAM map:\n"); | 1243 | pr_info("BIOS-provided physical RAM map:\n"); |
| 1242 | e820__print_table(who); | 1244 | e820__print_table(who); |
| 1243 | } | 1245 | } |
| 1244 | 1246 | ||
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c index bae0d32e327b..da5d8ac60062 100644 --- a/arch/x86/kernel/early-quirks.c +++ b/arch/x86/kernel/early-quirks.c | |||
| @@ -28,8 +28,6 @@ | |||
| 28 | #include <asm/irq_remapping.h> | 28 | #include <asm/irq_remapping.h> |
| 29 | #include <asm/early_ioremap.h> | 29 | #include <asm/early_ioremap.h> |
| 30 | 30 | ||
| 31 | #define dev_err(msg) pr_err("pci 0000:%02x:%02x.%d: %s", bus, slot, func, msg) | ||
| 32 | |||
| 33 | static void __init fix_hypertransport_config(int num, int slot, int func) | 31 | static void __init fix_hypertransport_config(int num, int slot, int func) |
| 34 | { | 32 | { |
| 35 | u32 htcfg; | 33 | u32 htcfg; |
| @@ -617,7 +615,8 @@ static void __init apple_airport_reset(int bus, int slot, int func) | |||
| 617 | 615 | ||
| 618 | pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); | 616 | pmcsr = read_pci_config_16(bus, slot, func, BCM4331_PM_CAP + PCI_PM_CTRL); |
| 619 | if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { | 617 | if ((pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0) { |
| 620 | dev_err("Cannot power up Apple AirPort card\n"); | 618 | pr_err("pci 0000:%02x:%02x.%d: Cannot power up Apple AirPort card\n", |
| 619 | bus, slot, func); | ||
| 621 | return; | 620 | return; |
| 622 | } | 621 | } |
| 623 | } | 622 | } |
| @@ -628,7 +627,8 @@ static void __init apple_airport_reset(int bus, int slot, int func) | |||
| 628 | 627 | ||
| 629 | mmio = early_ioremap(addr, BCM4331_MMIO_SIZE); | 628 | mmio = early_ioremap(addr, BCM4331_MMIO_SIZE); |
| 630 | if (!mmio) { | 629 | if (!mmio) { |
| 631 | dev_err("Cannot iomap Apple AirPort card\n"); | 630 | pr_err("pci 0000:%02x:%02x.%d: Cannot iomap Apple AirPort card\n", |
| 631 | bus, slot, func); | ||
| 632 | return; | 632 | return; |
| 633 | } | 633 | } |
| 634 | 634 | ||
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 8ce4212e2b8d..b6be34ee88e9 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c | |||
| @@ -975,8 +975,7 @@ int __init hpet_enable(void) | |||
| 975 | cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY); | 975 | cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY); |
| 976 | hpet_writel(cfg, HPET_CFG); | 976 | hpet_writel(cfg, HPET_CFG); |
| 977 | if (cfg) | 977 | if (cfg) |
| 978 | pr_warn("HPET: Unrecognized bits %#x set in global cfg\n", | 978 | pr_warn("Unrecognized bits %#x set in global cfg\n", cfg); |
| 979 | cfg); | ||
| 980 | 979 | ||
| 981 | for (i = 0; i <= last; ++i) { | 980 | for (i = 0; i <= last; ++i) { |
| 982 | cfg = hpet_readl(HPET_Tn_CFG(i)); | 981 | cfg = hpet_readl(HPET_Tn_CFG(i)); |
| @@ -988,7 +987,7 @@ int __init hpet_enable(void) | |||
| 988 | | HPET_TN_64BIT_CAP | HPET_TN_32BIT | HPET_TN_ROUTE | 987 | | HPET_TN_64BIT_CAP | HPET_TN_32BIT | HPET_TN_ROUTE |
| 989 | | HPET_TN_FSB | HPET_TN_FSB_CAP); | 988 | | HPET_TN_FSB | HPET_TN_FSB_CAP); |
| 990 | if (cfg) | 989 | if (cfg) |
| 991 | pr_warn("HPET: Unrecognized bits %#x set in cfg#%u\n", | 990 | pr_warn("Unrecognized bits %#x set in cfg#%u\n", |
| 992 | cfg, i); | 991 | cfg, i); |
| 993 | } | 992 | } |
| 994 | hpet_print_config(); | 993 | hpet_print_config(); |
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index c84bb5396958..58d8d800875d 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c | |||
| @@ -1083,8 +1083,8 @@ arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs | |||
| 1083 | return orig_ret_vaddr; | 1083 | return orig_ret_vaddr; |
| 1084 | 1084 | ||
| 1085 | if (nleft != rasize) { | 1085 | if (nleft != rasize) { |
| 1086 | pr_err("uprobe: return address clobbered: pid=%d, %%sp=%#lx, " | 1086 | pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n", |
| 1087 | "%%ip=%#lx\n", current->pid, regs->sp, regs->ip); | 1087 | current->pid, regs->sp, regs->ip); |
| 1088 | 1088 | ||
| 1089 | force_sig_info(SIGSEGV, SEND_SIG_FORCED, current); | 1089 | force_sig_info(SIGSEGV, SEND_SIG_FORCED, current); |
| 1090 | } | 1090 | } |
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 25504d5aa816..fa150855647c 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c | |||
| @@ -136,13 +136,13 @@ static int __init numa_add_memblk_to(int nid, u64 start, u64 end, | |||
| 136 | 136 | ||
| 137 | /* whine about and ignore invalid blks */ | 137 | /* whine about and ignore invalid blks */ |
| 138 | if (start > end || nid < 0 || nid >= MAX_NUMNODES) { | 138 | if (start > end || nid < 0 || nid >= MAX_NUMNODES) { |
| 139 | pr_warning("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", | 139 | pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", |
| 140 | nid, start, end - 1); | 140 | nid, start, end - 1); |
| 141 | return 0; | 141 | return 0; |
| 142 | } | 142 | } |
| 143 | 143 | ||
| 144 | if (mi->nr_blks >= NR_NODE_MEMBLKS) { | 144 | if (mi->nr_blks >= NR_NODE_MEMBLKS) { |
| 145 | pr_err("NUMA: too many memblk ranges\n"); | 145 | pr_err("too many memblk ranges\n"); |
| 146 | return -EINVAL; | 146 | return -EINVAL; |
| 147 | } | 147 | } |
| 148 | 148 | ||
| @@ -267,14 +267,14 @@ int __init numa_cleanup_meminfo(struct numa_meminfo *mi) | |||
| 267 | */ | 267 | */ |
| 268 | if (bi->end > bj->start && bi->start < bj->end) { | 268 | if (bi->end > bj->start && bi->start < bj->end) { |
| 269 | if (bi->nid != bj->nid) { | 269 | if (bi->nid != bj->nid) { |
| 270 | pr_err("NUMA: node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n", | 270 | pr_err("node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n", |
| 271 | bi->nid, bi->start, bi->end - 1, | 271 | bi->nid, bi->start, bi->end - 1, |
| 272 | bj->nid, bj->start, bj->end - 1); | 272 | bj->nid, bj->start, bj->end - 1); |
| 273 | return -EINVAL; | 273 | return -EINVAL; |
| 274 | } | 274 | } |
| 275 | pr_warning("NUMA: Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n", | 275 | pr_warn("Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n", |
| 276 | bi->nid, bi->start, bi->end - 1, | 276 | bi->nid, bi->start, bi->end - 1, |
| 277 | bj->start, bj->end - 1); | 277 | bj->start, bj->end - 1); |
| 278 | } | 278 | } |
| 279 | 279 | ||
| 280 | /* | 280 | /* |
| @@ -364,7 +364,7 @@ static int __init numa_alloc_distance(void) | |||
| 364 | phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), | 364 | phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), |
| 365 | size, PAGE_SIZE); | 365 | size, PAGE_SIZE); |
| 366 | if (!phys) { | 366 | if (!phys) { |
| 367 | pr_warning("NUMA: Warning: can't allocate distance table!\n"); | 367 | pr_warn("Warning: can't allocate distance table!\n"); |
| 368 | /* don't retry until explicitly reset */ | 368 | /* don't retry until explicitly reset */ |
| 369 | numa_distance = (void *)1LU; | 369 | numa_distance = (void *)1LU; |
| 370 | return -ENOMEM; | 370 | return -ENOMEM; |
| @@ -410,14 +410,14 @@ void __init numa_set_distance(int from, int to, int distance) | |||
| 410 | 410 | ||
| 411 | if (from >= numa_distance_cnt || to >= numa_distance_cnt || | 411 | if (from >= numa_distance_cnt || to >= numa_distance_cnt || |
| 412 | from < 0 || to < 0) { | 412 | from < 0 || to < 0) { |
| 413 | pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n", | 413 | pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n", |
| 414 | from, to, distance); | 414 | from, to, distance); |
| 415 | return; | 415 | return; |
| 416 | } | 416 | } |
| 417 | 417 | ||
| 418 | if ((u8)distance != distance || | 418 | if ((u8)distance != distance || |
| 419 | (from == to && distance != LOCAL_DISTANCE)) { | 419 | (from == to && distance != LOCAL_DISTANCE)) { |
| 420 | pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n", | 420 | pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n", |
| 421 | from, to, distance); | 421 | from, to, distance); |
| 422 | return; | 422 | return; |
| 423 | } | 423 | } |
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 263c8453815e..d765acedc05c 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c | |||
| @@ -1,4 +1,5 @@ | |||
| 1 | /* bpf_jit_comp.c : BPF JIT compiler | 1 | /* |
| 2 | * bpf_jit_comp.c: BPF JIT compiler | ||
| 2 | * | 3 | * |
| 3 | * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) | 4 | * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com) |
| 4 | * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com | 5 | * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
| @@ -17,7 +18,7 @@ | |||
| 17 | #include <asm/nospec-branch.h> | 18 | #include <asm/nospec-branch.h> |
| 18 | 19 | ||
| 19 | /* | 20 | /* |
| 20 | * assembly code in arch/x86/net/bpf_jit.S | 21 | * Assembly code in arch/x86/net/bpf_jit.S |
| 21 | */ | 22 | */ |
| 22 | extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; | 23 | extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; |
| 23 | extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[]; | 24 | extern u8 sk_load_word_positive_offset[], sk_load_half_positive_offset[]; |
| @@ -45,14 +46,15 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) | |||
| 45 | #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) | 46 | #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2) |
| 46 | #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) | 47 | #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3) |
| 47 | #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) | 48 | #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4) |
| 49 | |||
| 48 | #define EMIT1_off32(b1, off) \ | 50 | #define EMIT1_off32(b1, off) \ |
| 49 | do {EMIT1(b1); EMIT(off, 4); } while (0) | 51 | do { EMIT1(b1); EMIT(off, 4); } while (0) |
| 50 | #define EMIT2_off32(b1, b2, off) \ | 52 | #define EMIT2_off32(b1, b2, off) \ |
| 51 | do {EMIT2(b1, b2); EMIT(off, 4); } while (0) | 53 | do { EMIT2(b1, b2); EMIT(off, 4); } while (0) |
| 52 | #define EMIT3_off32(b1, b2, b3, off) \ | 54 | #define EMIT3_off32(b1, b2, b3, off) \ |
| 53 | do {EMIT3(b1, b2, b3); EMIT(off, 4); } while (0) | 55 | do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0) |
| 54 | #define EMIT4_off32(b1, b2, b3, b4, off) \ | 56 | #define EMIT4_off32(b1, b2, b3, b4, off) \ |
| 55 | do {EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) | 57 | do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0) |
| 56 | 58 | ||
| 57 | static bool is_imm8(int value) | 59 | static bool is_imm8(int value) |
| 58 | { | 60 | { |
| @@ -70,9 +72,10 @@ static bool is_uimm32(u64 value) | |||
| 70 | } | 72 | } |
| 71 | 73 | ||
| 72 | /* mov dst, src */ | 74 | /* mov dst, src */ |
| 73 | #define EMIT_mov(DST, SRC) \ | 75 | #define EMIT_mov(DST, SRC) \ |
| 74 | do {if (DST != SRC) \ | 76 | do { \ |
| 75 | EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \ | 77 | if (DST != SRC) \ |
| 78 | EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \ | ||
| 76 | } while (0) | 79 | } while (0) |
| 77 | 80 | ||
| 78 | static int bpf_size_to_x86_bytes(int bpf_size) | 81 | static int bpf_size_to_x86_bytes(int bpf_size) |
| @@ -89,7 +92,8 @@ static int bpf_size_to_x86_bytes(int bpf_size) | |||
| 89 | return 0; | 92 | return 0; |
| 90 | } | 93 | } |
| 91 | 94 | ||
| 92 | /* list of x86 cond jumps opcodes (. + s8) | 95 | /* |
| 96 | * List of x86 cond jumps opcodes (. + s8) | ||
| 93 | * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32) | 97 | * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32) |
| 94 | */ | 98 | */ |
| 95 | #define X86_JB 0x72 | 99 | #define X86_JB 0x72 |
| @@ -106,35 +110,37 @@ static int bpf_size_to_x86_bytes(int bpf_size) | |||
| 106 | #define CHOOSE_LOAD_FUNC(K, func) \ | 110 | #define CHOOSE_LOAD_FUNC(K, func) \ |
| 107 | ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) | 111 | ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) |
| 108 | 112 | ||
| 109 | /* pick a register outside of BPF range for JIT internal work */ | 113 | /* Pick a register outside of BPF range for JIT internal work */ |
| 110 | #define AUX_REG (MAX_BPF_JIT_REG + 1) | 114 | #define AUX_REG (MAX_BPF_JIT_REG + 1) |
| 111 | 115 | ||
| 112 | /* The following table maps BPF registers to x64 registers. | 116 | /* |
| 117 | * The following table maps BPF registers to x86-64 registers. | ||
| 113 | * | 118 | * |
| 114 | * x64 register r12 is unused, since if used as base address | 119 | * x86-64 register R12 is unused, since if used as base address |
| 115 | * register in load/store instructions, it always needs an | 120 | * register in load/store instructions, it always needs an |
| 116 | * extra byte of encoding and is callee saved. | 121 | * extra byte of encoding and is callee saved. |
| 117 | * | 122 | * |
| 118 | * r9 caches skb->len - skb->data_len | 123 | * R9 caches skb->len - skb->data_len |
| 119 | * r10 caches skb->data, and used for blinding (if enabled) | 124 | * R10 caches skb->data, and used for blinding (if enabled) |
| 120 | */ | 125 | */ |
| 121 | static const int reg2hex[] = { | 126 | static const int reg2hex[] = { |
| 122 | [BPF_REG_0] = 0, /* rax */ | 127 | [BPF_REG_0] = 0, /* RAX */ |
| 123 | [BPF_REG_1] = 7, /* rdi */ | 128 | [BPF_REG_1] = 7, /* RDI */ |
| 124 | [BPF_REG_2] = 6, /* rsi */ | 129 | [BPF_REG_2] = 6, /* RSI */ |
| 125 | [BPF_REG_3] = 2, /* rdx */ | 130 | [BPF_REG_3] = 2, /* RDX */ |
| 126 | [BPF_REG_4] = 1, /* rcx */ | 131 | [BPF_REG_4] = 1, /* RCX */ |
| 127 | [BPF_REG_5] = 0, /* r8 */ | 132 | [BPF_REG_5] = 0, /* R8 */ |
| 128 | [BPF_REG_6] = 3, /* rbx callee saved */ | 133 | [BPF_REG_6] = 3, /* RBX callee saved */ |
| 129 | [BPF_REG_7] = 5, /* r13 callee saved */ | 134 | [BPF_REG_7] = 5, /* R13 callee saved */ |
| 130 | [BPF_REG_8] = 6, /* r14 callee saved */ | 135 | [BPF_REG_8] = 6, /* R14 callee saved */ |
| 131 | [BPF_REG_9] = 7, /* r15 callee saved */ | 136 | [BPF_REG_9] = 7, /* R15 callee saved */ |
| 132 | [BPF_REG_FP] = 5, /* rbp readonly */ | 137 | [BPF_REG_FP] = 5, /* RBP readonly */ |
| 133 | [BPF_REG_AX] = 2, /* r10 temp register */ | 138 | [BPF_REG_AX] = 2, /* R10 temp register */ |
| 134 | [AUX_REG] = 3, /* r11 temp register */ | 139 | [AUX_REG] = 3, /* R11 temp register */ |
| 135 | }; | 140 | }; |
| 136 | 141 | ||
| 137 | /* is_ereg() == true if BPF register 'reg' maps to x64 r8..r15 | 142 | /* |
| 143 | * is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15 | ||
| 138 | * which need extra byte of encoding. | 144 | * which need extra byte of encoding. |
| 139 | * rax,rcx,...,rbp have simpler encoding | 145 | * rax,rcx,...,rbp have simpler encoding |
| 140 | */ | 146 | */ |
| @@ -153,7 +159,7 @@ static bool is_axreg(u32 reg) | |||
| 153 | return reg == BPF_REG_0; | 159 | return reg == BPF_REG_0; |
| 154 | } | 160 | } |
| 155 | 161 | ||
| 156 | /* add modifiers if 'reg' maps to x64 registers r8..r15 */ | 162 | /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */ |
| 157 | static u8 add_1mod(u8 byte, u32 reg) | 163 | static u8 add_1mod(u8 byte, u32 reg) |
| 158 | { | 164 | { |
| 159 | if (is_ereg(reg)) | 165 | if (is_ereg(reg)) |
| @@ -170,13 +176,13 @@ static u8 add_2mod(u8 byte, u32 r1, u32 r2) | |||
| 170 | return byte; | 176 | return byte; |
| 171 | } | 177 | } |
| 172 | 178 | ||
| 173 | /* encode 'dst_reg' register into x64 opcode 'byte' */ | 179 | /* Encode 'dst_reg' register into x86-64 opcode 'byte' */ |
| 174 | static u8 add_1reg(u8 byte, u32 dst_reg) | 180 | static u8 add_1reg(u8 byte, u32 dst_reg) |
| 175 | { | 181 | { |
| 176 | return byte + reg2hex[dst_reg]; | 182 | return byte + reg2hex[dst_reg]; |
| 177 | } | 183 | } |
| 178 | 184 | ||
| 179 | /* encode 'dst_reg' and 'src_reg' registers into x64 opcode 'byte' */ | 185 | /* Encode 'dst_reg' and 'src_reg' registers into x86-64 opcode 'byte' */ |
| 180 | static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) | 186 | static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) |
| 181 | { | 187 | { |
| 182 | return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3); | 188 | return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3); |
| @@ -184,27 +190,28 @@ static u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg) | |||
| 184 | 190 | ||
| 185 | static void jit_fill_hole(void *area, unsigned int size) | 191 | static void jit_fill_hole(void *area, unsigned int size) |
| 186 | { | 192 | { |
| 187 | /* fill whole space with int3 instructions */ | 193 | /* Fill whole space with INT3 instructions */ |
| 188 | memset(area, 0xcc, size); | 194 | memset(area, 0xcc, size); |
| 189 | } | 195 | } |
| 190 | 196 | ||
| 191 | struct jit_context { | 197 | struct jit_context { |
| 192 | int cleanup_addr; /* epilogue code offset */ | 198 | int cleanup_addr; /* Epilogue code offset */ |
| 193 | bool seen_ld_abs; | 199 | bool seen_ld_abs; |
| 194 | bool seen_ax_reg; | 200 | bool seen_ax_reg; |
| 195 | }; | 201 | }; |
| 196 | 202 | ||
| 197 | /* maximum number of bytes emitted while JITing one eBPF insn */ | 203 | /* Maximum number of bytes emitted while JITing one eBPF insn */ |
| 198 | #define BPF_MAX_INSN_SIZE 128 | 204 | #define BPF_MAX_INSN_SIZE 128 |
| 199 | #define BPF_INSN_SAFETY 64 | 205 | #define BPF_INSN_SAFETY 64 |
| 200 | 206 | ||
| 201 | #define AUX_STACK_SPACE \ | 207 | #define AUX_STACK_SPACE \ |
| 202 | (32 /* space for rbx, r13, r14, r15 */ + \ | 208 | (32 /* Space for RBX, R13, R14, R15 */ + \ |
| 203 | 8 /* space for skb_copy_bits() buffer */) | 209 | 8 /* Space for skb_copy_bits() buffer */) |
| 204 | 210 | ||
| 205 | #define PROLOGUE_SIZE 37 | 211 | #define PROLOGUE_SIZE 37 |
| 206 | 212 | ||
| 207 | /* emit x64 prologue code for BPF program and check it's size. | 213 | /* |
| 214 | * Emit x86-64 prologue code for BPF program and check its size. | ||
| 208 | * bpf_tail_call helper will skip it while jumping into another program | 215 | * bpf_tail_call helper will skip it while jumping into another program |
| 209 | */ | 216 | */ |
| 210 | static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | 217 | static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) |
| @@ -212,8 +219,11 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
| 212 | u8 *prog = *pprog; | 219 | u8 *prog = *pprog; |
| 213 | int cnt = 0; | 220 | int cnt = 0; |
| 214 | 221 | ||
| 215 | EMIT1(0x55); /* push rbp */ | 222 | /* push rbp */ |
| 216 | EMIT3(0x48, 0x89, 0xE5); /* mov rbp,rsp */ | 223 | EMIT1(0x55); |
| 224 | |||
| 225 | /* mov rbp,rsp */ | ||
| 226 | EMIT3(0x48, 0x89, 0xE5); | ||
| 217 | 227 | ||
| 218 | /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */ | 228 | /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */ |
| 219 | EMIT3_off32(0x48, 0x81, 0xEC, | 229 | EMIT3_off32(0x48, 0x81, 0xEC, |
| @@ -222,14 +232,15 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
| 222 | /* sub rbp, AUX_STACK_SPACE */ | 232 | /* sub rbp, AUX_STACK_SPACE */ |
| 223 | EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE); | 233 | EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE); |
| 224 | 234 | ||
| 225 | /* all classic BPF filters use R6(rbx) save it */ | 235 | /* All classic BPF filters use R6(rbx) save it */ |
| 226 | 236 | ||
| 227 | /* mov qword ptr [rbp+0],rbx */ | 237 | /* mov qword ptr [rbp+0],rbx */ |
| 228 | EMIT4(0x48, 0x89, 0x5D, 0); | 238 | EMIT4(0x48, 0x89, 0x5D, 0); |
| 229 | 239 | ||
| 230 | /* bpf_convert_filter() maps classic BPF register X to R7 and uses R8 | 240 | /* |
| 231 | * as temporary, so all tcpdump filters need to spill/fill R7(r13) and | 241 | * bpf_convert_filter() maps classic BPF register X to R7 and uses R8 |
| 232 | * R8(r14). R9(r15) spill could be made conditional, but there is only | 242 | * as temporary, so all tcpdump filters need to spill/fill R7(R13) and |
| 243 | * R8(R14). R9(R15) spill could be made conditional, but there is only | ||
| 233 | * one 'bpf_error' return path out of helper functions inside bpf_jit.S | 244 | * one 'bpf_error' return path out of helper functions inside bpf_jit.S |
| 234 | * The overhead of extra spill is negligible for any filter other | 245 | * The overhead of extra spill is negligible for any filter other |
| 235 | * than synthetic ones. Therefore not worth adding complexity. | 246 | * than synthetic ones. Therefore not worth adding complexity. |
| @@ -243,9 +254,10 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
| 243 | EMIT4(0x4C, 0x89, 0x7D, 24); | 254 | EMIT4(0x4C, 0x89, 0x7D, 24); |
| 244 | 255 | ||
| 245 | if (!ebpf_from_cbpf) { | 256 | if (!ebpf_from_cbpf) { |
| 246 | /* Clear the tail call counter (tail_call_cnt): for eBPF tail | 257 | /* |
| 258 | * Clear the tail call counter (tail_call_cnt): for eBPF tail | ||
| 247 | * calls we need to reset the counter to 0. It's done in two | 259 | * calls we need to reset the counter to 0. It's done in two |
| 248 | * instructions, resetting rax register to 0, and moving it | 260 | * instructions, resetting RAX register to 0, and moving it |
| 249 | * to the counter location. | 261 | * to the counter location. |
| 250 | */ | 262 | */ |
| 251 | 263 | ||
| @@ -260,7 +272,9 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf) | |||
| 260 | *pprog = prog; | 272 | *pprog = prog; |
| 261 | } | 273 | } |
| 262 | 274 | ||
| 263 | /* generate the following code: | 275 | /* |
| 276 | * Generate the following code: | ||
| 277 | * | ||
| 264 | * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... | 278 | * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ... |
| 265 | * if (index >= array->map.max_entries) | 279 | * if (index >= array->map.max_entries) |
| 266 | * goto out; | 280 | * goto out; |
| @@ -278,23 +292,26 @@ static void emit_bpf_tail_call(u8 **pprog) | |||
| 278 | int label1, label2, label3; | 292 | int label1, label2, label3; |
| 279 | int cnt = 0; | 293 | int cnt = 0; |
| 280 | 294 | ||
| 281 | /* rdi - pointer to ctx | 295 | /* |
| 296 | * rdi - pointer to ctx | ||
| 282 | * rsi - pointer to bpf_array | 297 | * rsi - pointer to bpf_array |
| 283 | * rdx - index in bpf_array | 298 | * rdx - index in bpf_array |
| 284 | */ | 299 | */ |
| 285 | 300 | ||
| 286 | /* if (index >= array->map.max_entries) | 301 | /* |
| 287 | * goto out; | 302 | * if (index >= array->map.max_entries) |
| 303 | * goto out; | ||
| 288 | */ | 304 | */ |
| 289 | EMIT2(0x89, 0xD2); /* mov edx, edx */ | 305 | EMIT2(0x89, 0xD2); /* mov edx, edx */ |
| 290 | EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ | 306 | EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ |
| 291 | offsetof(struct bpf_array, map.max_entries)); | 307 | offsetof(struct bpf_array, map.max_entries)); |
| 292 | #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* number of bytes to jump */ | 308 | #define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */ |
| 293 | EMIT2(X86_JBE, OFFSET1); /* jbe out */ | 309 | EMIT2(X86_JBE, OFFSET1); /* jbe out */ |
| 294 | label1 = cnt; | 310 | label1 = cnt; |
| 295 | 311 | ||
| 296 | /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) | 312 | /* |
| 297 | * goto out; | 313 | * if (tail_call_cnt > MAX_TAIL_CALL_CNT) |
| 314 | * goto out; | ||
| 298 | */ | 315 | */ |
| 299 | EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */ | 316 | EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */ |
| 300 | EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ | 317 | EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */ |
| @@ -308,8 +325,9 @@ static void emit_bpf_tail_call(u8 **pprog) | |||
| 308 | EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ | 325 | EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */ |
| 309 | offsetof(struct bpf_array, ptrs)); | 326 | offsetof(struct bpf_array, ptrs)); |
| 310 | 327 | ||
| 311 | /* if (prog == NULL) | 328 | /* |
| 312 | * goto out; | 329 | * if (prog == NULL) |
| 330 | * goto out; | ||
| 313 | */ | 331 | */ |
| 314 | EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ | 332 | EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ |
| 315 | #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE) | 333 | #define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE) |
| @@ -321,7 +339,8 @@ static void emit_bpf_tail_call(u8 **pprog) | |||
| 321 | offsetof(struct bpf_prog, bpf_func)); | 339 | offsetof(struct bpf_prog, bpf_func)); |
| 322 | EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ | 340 | EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE); /* add rax, prologue_size */ |
| 323 | 341 | ||
| 324 | /* now we're ready to jump into next BPF program | 342 | /* |
| 343 | * Wow we're ready to jump into next BPF program | ||
| 325 | * rdi == ctx (1st arg) | 344 | * rdi == ctx (1st arg) |
| 326 | * rax == prog->bpf_func + prologue_size | 345 | * rax == prog->bpf_func + prologue_size |
| 327 | */ | 346 | */ |
| @@ -340,7 +359,8 @@ static void emit_load_skb_data_hlen(u8 **pprog) | |||
| 340 | u8 *prog = *pprog; | 359 | u8 *prog = *pprog; |
| 341 | int cnt = 0; | 360 | int cnt = 0; |
| 342 | 361 | ||
| 343 | /* r9d = skb->len - skb->data_len (headlen) | 362 | /* |
| 363 | * r9d = skb->len - skb->data_len (headlen) | ||
| 344 | * r10 = skb->data | 364 | * r10 = skb->data |
| 345 | */ | 365 | */ |
| 346 | /* mov %r9d, off32(%rdi) */ | 366 | /* mov %r9d, off32(%rdi) */ |
| @@ -361,7 +381,8 @@ static void emit_mov_imm32(u8 **pprog, bool sign_propagate, | |||
| 361 | u8 b1, b2, b3; | 381 | u8 b1, b2, b3; |
| 362 | int cnt = 0; | 382 | int cnt = 0; |
| 363 | 383 | ||
| 364 | /* optimization: if imm32 is positive, use 'mov %eax, imm32' | 384 | /* |
| 385 | * Optimization: if imm32 is positive, use 'mov %eax, imm32' | ||
| 365 | * (which zero-extends imm32) to save 2 bytes. | 386 | * (which zero-extends imm32) to save 2 bytes. |
| 366 | */ | 387 | */ |
| 367 | if (sign_propagate && (s32)imm32 < 0) { | 388 | if (sign_propagate && (s32)imm32 < 0) { |
| @@ -373,7 +394,8 @@ static void emit_mov_imm32(u8 **pprog, bool sign_propagate, | |||
| 373 | goto done; | 394 | goto done; |
| 374 | } | 395 | } |
| 375 | 396 | ||
| 376 | /* optimization: if imm32 is zero, use 'xor %eax, %eax' | 397 | /* |
| 398 | * Optimization: if imm32 is zero, use 'xor %eax, %eax' | ||
| 377 | * to save 3 bytes. | 399 | * to save 3 bytes. |
| 378 | */ | 400 | */ |
| 379 | if (imm32 == 0) { | 401 | if (imm32 == 0) { |
| @@ -400,7 +422,8 @@ static void emit_mov_imm64(u8 **pprog, u32 dst_reg, | |||
| 400 | int cnt = 0; | 422 | int cnt = 0; |
| 401 | 423 | ||
| 402 | if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) { | 424 | if (is_uimm32(((u64)imm32_hi << 32) | (u32)imm32_lo)) { |
| 403 | /* For emitting plain u32, where sign bit must not be | 425 | /* |
| 426 | * For emitting plain u32, where sign bit must not be | ||
| 404 | * propagated LLVM tends to load imm64 over mov32 | 427 | * propagated LLVM tends to load imm64 over mov32 |
| 405 | * directly, so save couple of bytes by just doing | 428 | * directly, so save couple of bytes by just doing |
| 406 | * 'mov %eax, imm32' instead. | 429 | * 'mov %eax, imm32' instead. |
| @@ -525,7 +548,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 525 | else if (is_ereg(dst_reg)) | 548 | else if (is_ereg(dst_reg)) |
| 526 | EMIT1(add_1mod(0x40, dst_reg)); | 549 | EMIT1(add_1mod(0x40, dst_reg)); |
| 527 | 550 | ||
| 528 | /* b3 holds 'normal' opcode, b2 short form only valid | 551 | /* |
| 552 | * b3 holds 'normal' opcode, b2 short form only valid | ||
| 529 | * in case dst is eax/rax. | 553 | * in case dst is eax/rax. |
| 530 | */ | 554 | */ |
| 531 | switch (BPF_OP(insn->code)) { | 555 | switch (BPF_OP(insn->code)) { |
| @@ -593,7 +617,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 593 | /* mov rax, dst_reg */ | 617 | /* mov rax, dst_reg */ |
| 594 | EMIT_mov(BPF_REG_0, dst_reg); | 618 | EMIT_mov(BPF_REG_0, dst_reg); |
| 595 | 619 | ||
| 596 | /* xor edx, edx | 620 | /* |
| 621 | * xor edx, edx | ||
| 597 | * equivalent to 'xor rdx, rdx', but one byte less | 622 | * equivalent to 'xor rdx, rdx', but one byte less |
| 598 | */ | 623 | */ |
| 599 | EMIT2(0x31, 0xd2); | 624 | EMIT2(0x31, 0xd2); |
| @@ -655,7 +680,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 655 | } | 680 | } |
| 656 | break; | 681 | break; |
| 657 | } | 682 | } |
| 658 | /* shifts */ | 683 | /* Shifts */ |
| 659 | case BPF_ALU | BPF_LSH | BPF_K: | 684 | case BPF_ALU | BPF_LSH | BPF_K: |
| 660 | case BPF_ALU | BPF_RSH | BPF_K: | 685 | case BPF_ALU | BPF_RSH | BPF_K: |
| 661 | case BPF_ALU | BPF_ARSH | BPF_K: | 686 | case BPF_ALU | BPF_ARSH | BPF_K: |
| @@ -686,7 +711,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 686 | case BPF_ALU64 | BPF_RSH | BPF_X: | 711 | case BPF_ALU64 | BPF_RSH | BPF_X: |
| 687 | case BPF_ALU64 | BPF_ARSH | BPF_X: | 712 | case BPF_ALU64 | BPF_ARSH | BPF_X: |
| 688 | 713 | ||
| 689 | /* check for bad case when dst_reg == rcx */ | 714 | /* Check for bad case when dst_reg == rcx */ |
| 690 | if (dst_reg == BPF_REG_4) { | 715 | if (dst_reg == BPF_REG_4) { |
| 691 | /* mov r11, dst_reg */ | 716 | /* mov r11, dst_reg */ |
| 692 | EMIT_mov(AUX_REG, dst_reg); | 717 | EMIT_mov(AUX_REG, dst_reg); |
| @@ -724,13 +749,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 724 | case BPF_ALU | BPF_END | BPF_FROM_BE: | 749 | case BPF_ALU | BPF_END | BPF_FROM_BE: |
| 725 | switch (imm32) { | 750 | switch (imm32) { |
| 726 | case 16: | 751 | case 16: |
| 727 | /* emit 'ror %ax, 8' to swap lower 2 bytes */ | 752 | /* Emit 'ror %ax, 8' to swap lower 2 bytes */ |
| 728 | EMIT1(0x66); | 753 | EMIT1(0x66); |
| 729 | if (is_ereg(dst_reg)) | 754 | if (is_ereg(dst_reg)) |
| 730 | EMIT1(0x41); | 755 | EMIT1(0x41); |
| 731 | EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); | 756 | EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); |
| 732 | 757 | ||
| 733 | /* emit 'movzwl eax, ax' */ | 758 | /* Emit 'movzwl eax, ax' */ |
| 734 | if (is_ereg(dst_reg)) | 759 | if (is_ereg(dst_reg)) |
| 735 | EMIT3(0x45, 0x0F, 0xB7); | 760 | EMIT3(0x45, 0x0F, 0xB7); |
| 736 | else | 761 | else |
| @@ -738,7 +763,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 738 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); | 763 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); |
| 739 | break; | 764 | break; |
| 740 | case 32: | 765 | case 32: |
| 741 | /* emit 'bswap eax' to swap lower 4 bytes */ | 766 | /* Emit 'bswap eax' to swap lower 4 bytes */ |
| 742 | if (is_ereg(dst_reg)) | 767 | if (is_ereg(dst_reg)) |
| 743 | EMIT2(0x41, 0x0F); | 768 | EMIT2(0x41, 0x0F); |
| 744 | else | 769 | else |
| @@ -746,7 +771,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 746 | EMIT1(add_1reg(0xC8, dst_reg)); | 771 | EMIT1(add_1reg(0xC8, dst_reg)); |
| 747 | break; | 772 | break; |
| 748 | case 64: | 773 | case 64: |
| 749 | /* emit 'bswap rax' to swap 8 bytes */ | 774 | /* Emit 'bswap rax' to swap 8 bytes */ |
| 750 | EMIT3(add_1mod(0x48, dst_reg), 0x0F, | 775 | EMIT3(add_1mod(0x48, dst_reg), 0x0F, |
| 751 | add_1reg(0xC8, dst_reg)); | 776 | add_1reg(0xC8, dst_reg)); |
| 752 | break; | 777 | break; |
| @@ -756,7 +781,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 756 | case BPF_ALU | BPF_END | BPF_FROM_LE: | 781 | case BPF_ALU | BPF_END | BPF_FROM_LE: |
| 757 | switch (imm32) { | 782 | switch (imm32) { |
| 758 | case 16: | 783 | case 16: |
| 759 | /* emit 'movzwl eax, ax' to zero extend 16-bit | 784 | /* |
| 785 | * Emit 'movzwl eax, ax' to zero extend 16-bit | ||
| 760 | * into 64 bit | 786 | * into 64 bit |
| 761 | */ | 787 | */ |
| 762 | if (is_ereg(dst_reg)) | 788 | if (is_ereg(dst_reg)) |
| @@ -766,7 +792,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, | |||
| 766 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); | 792 | EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); |
| 767 | break; | 793 | break; |
| 768 | case 32: | 794 | case 32: |
| 769 | /* emit 'mov eax, eax' to clear upper 32-bits */ | 795 | /* Emit 'mov eax, eax' to clear upper 32-bits */ |
| 770 | if (is_ereg(dst_reg)) | 796 | if (is_ereg(dst_reg)) |
| 771 | EMIT1(0x45); | 797 | EMIT1(0x45); |
| 772 | EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); | 798 | EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); |
| @@ -809,9 +835,9 @@ st: if (is_imm8(insn->off)) | |||
| 809 | 835 | ||
| 810 | /* STX: *(u8*)(dst_reg + off) = src_reg */ | 836 | /* STX: *(u8*)(dst_reg + off) = src_reg */ |
| 811 | case BPF_STX | BPF_MEM | BPF_B: | 837 | case BPF_STX | BPF_MEM | BPF_B: |
| 812 | /* emit 'mov byte ptr [rax + off], al' */ | 838 | /* Emit 'mov byte ptr [rax + off], al' */ |
| 813 | if (is_ereg(dst_reg) || is_ereg(src_reg) || | 839 | if (is_ereg(dst_reg) || is_ereg(src_reg) || |
| 814 | /* have to add extra byte for x86 SIL, DIL regs */ | 840 | /* We have to add extra byte for x86 SIL, DIL regs */ |
| 815 | src_reg == BPF_REG_1 || src_reg == BPF_REG_2) | 841 | src_reg == BPF_REG_1 || src_reg == BPF_REG_2) |
| 816 | EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); | 842 | EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); |
| 817 | else | 843 | else |
| @@ -840,25 +866,26 @@ stx: if (is_imm8(insn->off)) | |||
| 840 | 866 | ||
| 841 | /* LDX: dst_reg = *(u8*)(src_reg + off) */ | 867 | /* LDX: dst_reg = *(u8*)(src_reg + off) */ |
| 842 | case BPF_LDX | BPF_MEM | BPF_B: | 868 | case BPF_LDX | BPF_MEM | BPF_B: |
| 843 | /* emit 'movzx rax, byte ptr [rax + off]' */ | 869 | /* Emit 'movzx rax, byte ptr [rax + off]' */ |
| 844 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); | 870 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); |
| 845 | goto ldx; | 871 | goto ldx; |
| 846 | case BPF_LDX | BPF_MEM | BPF_H: | 872 | case BPF_LDX | BPF_MEM | BPF_H: |
| 847 | /* emit 'movzx rax, word ptr [rax + off]' */ | 873 | /* Emit 'movzx rax, word ptr [rax + off]' */ |
| 848 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); | 874 | EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); |
| 849 | goto ldx; | 875 | goto ldx; |
| 850 | case BPF_LDX | BPF_MEM | BPF_W: | 876 | case BPF_LDX | BPF_MEM | BPF_W: |
| 851 | /* emit 'mov eax, dword ptr [rax+0x14]' */ | 877 | /* Emit 'mov eax, dword ptr [rax+0x14]' */ |
| 852 | if (is_ereg(dst_reg) || is_ereg(src_reg)) | 878 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 853 | EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); | 879 | EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); |
| 854 | else | 880 | else |
| 855 | EMIT1(0x8B); | 881 | EMIT1(0x8B); |
| 856 | goto ldx; | 882 | goto ldx; |
| 857 | case BPF_LDX | BPF_MEM | BPF_DW: | 883 | case BPF_LDX | BPF_MEM | BPF_DW: |
| 858 | /* emit 'mov rax, qword ptr [rax+0x14]' */ | 884 | /* Emit 'mov rax, qword ptr [rax+0x14]' */ |
| 859 | EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); | 885 | EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); |
| 860 | ldx: /* if insn->off == 0 we can save one extra byte, but | 886 | ldx: /* |
| 861 | * special case of x86 r13 which always needs an offset | 887 | * If insn->off == 0 we can save one extra byte, but |
| 888 | * special case of x86 R13 which always needs an offset | ||
| 862 | * is not worth the hassle | 889 | * is not worth the hassle |
| 863 | */ | 890 | */ |
| 864 | if (is_imm8(insn->off)) | 891 | if (is_imm8(insn->off)) |
| @@ -870,7 +897,7 @@ ldx: /* if insn->off == 0 we can save one extra byte, but | |||
| 870 | 897 | ||
| 871 | /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */ | 898 | /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */ |
| 872 | case BPF_STX | BPF_XADD | BPF_W: | 899 | case BPF_STX | BPF_XADD | BPF_W: |
| 873 | /* emit 'lock add dword ptr [rax + off], eax' */ | 900 | /* Emit 'lock add dword ptr [rax + off], eax' */ |
| 874 | if (is_ereg(dst_reg) || is_ereg(src_reg)) | 901 | if (is_ereg(dst_reg) || is_ereg(src_reg)) |
| 875 | EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); | 902 | EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01); |
| 876 | else | 903 | else |
| @@ -897,14 +924,15 @@ xadd: if (is_imm8(insn->off)) | |||
| 897 | } else { | 924 | } else { |
| 898 | EMIT2(0x41, 0x52); /* push %r10 */ | 925 | EMIT2(0x41, 0x52); /* push %r10 */ |
| 899 | EMIT2(0x41, 0x51); /* push %r9 */ | 926 | EMIT2(0x41, 0x51); /* push %r9 */ |
| 900 | /* need to adjust jmp offset, since | 927 | /* |
| 928 | * We need to adjust jmp offset, since | ||
| 901 | * pop %r9, pop %r10 take 4 bytes after call insn | 929 | * pop %r9, pop %r10 take 4 bytes after call insn |
| 902 | */ | 930 | */ |
| 903 | jmp_offset += 4; | 931 | jmp_offset += 4; |
| 904 | } | 932 | } |
| 905 | } | 933 | } |
| 906 | if (!imm32 || !is_simm32(jmp_offset)) { | 934 | if (!imm32 || !is_simm32(jmp_offset)) { |
| 907 | pr_err("unsupported bpf func %d addr %p image %p\n", | 935 | pr_err("unsupported BPF func %d addr %p image %p\n", |
| 908 | imm32, func, image); | 936 | imm32, func, image); |
| 909 | return -EINVAL; | 937 | return -EINVAL; |
| 910 | } | 938 | } |
| @@ -970,7 +998,7 @@ xadd: if (is_imm8(insn->off)) | |||
| 970 | else | 998 | else |
| 971 | EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); | 999 | EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); |
| 972 | 1000 | ||
| 973 | emit_cond_jmp: /* convert BPF opcode to x86 */ | 1001 | emit_cond_jmp: /* Convert BPF opcode to x86 */ |
| 974 | switch (BPF_OP(insn->code)) { | 1002 | switch (BPF_OP(insn->code)) { |
| 975 | case BPF_JEQ: | 1003 | case BPF_JEQ: |
| 976 | jmp_cond = X86_JE; | 1004 | jmp_cond = X86_JE; |
| @@ -996,22 +1024,22 @@ emit_cond_jmp: /* convert BPF opcode to x86 */ | |||
| 996 | jmp_cond = X86_JBE; | 1024 | jmp_cond = X86_JBE; |
| 997 | break; | 1025 | break; |
| 998 | case BPF_JSGT: | 1026 | case BPF_JSGT: |
| 999 | /* signed '>', GT in x86 */ | 1027 | /* Signed '>', GT in x86 */ |
| 1000 | jmp_cond = X86_JG; | 1028 | jmp_cond = X86_JG; |
| 1001 | break; | 1029 | break; |
| 1002 | case BPF_JSLT: | 1030 | case BPF_JSLT: |
| 1003 | /* signed '<', LT in x86 */ | 1031 | /* Signed '<', LT in x86 */ |
| 1004 | jmp_cond = X86_JL; | 1032 | jmp_cond = X86_JL; |
| 1005 | break; | 1033 | break; |
| 1006 | case BPF_JSGE: | 1034 | case BPF_JSGE: |
| 1007 | /* signed '>=', GE in x86 */ | 1035 | /* Signed '>=', GE in x86 */ |
| 1008 | jmp_cond = X86_JGE; | 1036 | jmp_cond = X86_JGE; |
| 1009 | break; | 1037 | break; |
| 1010 | case BPF_JSLE: | 1038 | case BPF_JSLE: |
| 1011 | /* signed '<=', LE in x86 */ | 1039 | /* Signed '<=', LE in x86 */ |
| 1012 | jmp_cond = X86_JLE; | 1040 | jmp_cond = X86_JLE; |
| 1013 | break; | 1041 | break; |
| 1014 | default: /* to silence gcc warning */ | 1042 | default: /* to silence GCC warning */ |
| 1015 | return -EFAULT; | 1043 | return -EFAULT; |
| 1016 | } | 1044 | } |
| 1017 | jmp_offset = addrs[i + insn->off] - addrs[i]; | 1045 | jmp_offset = addrs[i + insn->off] - addrs[i]; |
| @@ -1039,7 +1067,7 @@ emit_cond_jmp: /* convert BPF opcode to x86 */ | |||
| 1039 | jmp_offset = addrs[i + insn->off] - addrs[i]; | 1067 | jmp_offset = addrs[i + insn->off] - addrs[i]; |
| 1040 | 1068 | ||
| 1041 | if (!jmp_offset) | 1069 | if (!jmp_offset) |
| 1042 | /* optimize out nop jumps */ | 1070 | /* Optimize out nop jumps */ |
| 1043 | break; | 1071 | break; |
| 1044 | emit_jmp: | 1072 | emit_jmp: |
| 1045 | if (is_imm8(jmp_offset)) { | 1073 | if (is_imm8(jmp_offset)) { |
| @@ -1061,7 +1089,7 @@ common_load: | |||
| 1061 | ctx->seen_ld_abs = seen_ld_abs = true; | 1089 | ctx->seen_ld_abs = seen_ld_abs = true; |
| 1062 | jmp_offset = func - (image + addrs[i]); | 1090 | jmp_offset = func - (image + addrs[i]); |
| 1063 | if (!func || !is_simm32(jmp_offset)) { | 1091 | if (!func || !is_simm32(jmp_offset)) { |
| 1064 | pr_err("unsupported bpf func %d addr %p image %p\n", | 1092 | pr_err("unsupported BPF func %d addr %p image %p\n", |
| 1065 | imm32, func, image); | 1093 | imm32, func, image); |
| 1066 | return -EINVAL; | 1094 | return -EINVAL; |
| 1067 | } | 1095 | } |
| @@ -1080,7 +1108,8 @@ common_load: | |||
| 1080 | EMIT2_off32(0x81, 0xC6, imm32); | 1108 | EMIT2_off32(0x81, 0xC6, imm32); |
| 1081 | } | 1109 | } |
| 1082 | } | 1110 | } |
| 1083 | /* skb pointer is in R6 (%rbx), it will be copied into | 1111 | /* |
| 1112 | * skb pointer is in R6 (%rbx), it will be copied into | ||
| 1084 | * %rdi if skb_copy_bits() call is necessary. | 1113 | * %rdi if skb_copy_bits() call is necessary. |
| 1085 | * sk_load_* helpers also use %r10 and %r9d. | 1114 | * sk_load_* helpers also use %r10 and %r9d. |
| 1086 | * See bpf_jit.S | 1115 | * See bpf_jit.S |
| @@ -1111,7 +1140,7 @@ common_load: | |||
| 1111 | goto emit_jmp; | 1140 | goto emit_jmp; |
| 1112 | } | 1141 | } |
| 1113 | seen_exit = true; | 1142 | seen_exit = true; |
| 1114 | /* update cleanup_addr */ | 1143 | /* Update cleanup_addr */ |
| 1115 | ctx->cleanup_addr = proglen; | 1144 | ctx->cleanup_addr = proglen; |
| 1116 | /* mov rbx, qword ptr [rbp+0] */ | 1145 | /* mov rbx, qword ptr [rbp+0] */ |
| 1117 | EMIT4(0x48, 0x8B, 0x5D, 0); | 1146 | EMIT4(0x48, 0x8B, 0x5D, 0); |
| @@ -1129,10 +1158,11 @@ common_load: | |||
| 1129 | break; | 1158 | break; |
| 1130 | 1159 | ||
| 1131 | default: | 1160 | default: |
| 1132 | /* By design x64 JIT should support all BPF instructions | 1161 | /* |
| 1162 | * By design x86-64 JIT should support all BPF instructions. | ||
| 1133 | * This error will be seen if new instruction was added | 1163 | * This error will be seen if new instruction was added |
| 1134 | * to interpreter, but not to JIT | 1164 | * to the interpreter, but not to the JIT, or if there is |
| 1135 | * or if there is junk in bpf_prog | 1165 | * junk in bpf_prog. |
| 1136 | */ | 1166 | */ |
| 1137 | pr_err("bpf_jit: unknown opcode %02x\n", insn->code); | 1167 | pr_err("bpf_jit: unknown opcode %02x\n", insn->code); |
| 1138 | return -EINVAL; | 1168 | return -EINVAL; |
| @@ -1184,7 +1214,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) | |||
| 1184 | return orig_prog; | 1214 | return orig_prog; |
| 1185 | 1215 | ||
| 1186 | tmp = bpf_jit_blind_constants(prog); | 1216 | tmp = bpf_jit_blind_constants(prog); |
| 1187 | /* If blinding was requested and we failed during blinding, | 1217 | /* |
| 1218 | * If blinding was requested and we failed during blinding, | ||
| 1188 | * we must fall back to the interpreter. | 1219 | * we must fall back to the interpreter. |
| 1189 | */ | 1220 | */ |
| 1190 | if (IS_ERR(tmp)) | 1221 | if (IS_ERR(tmp)) |
| @@ -1218,8 +1249,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) | |||
| 1218 | goto out_addrs; | 1249 | goto out_addrs; |
| 1219 | } | 1250 | } |
| 1220 | 1251 | ||
| 1221 | /* Before first pass, make a rough estimation of addrs[] | 1252 | /* |
| 1222 | * each bpf instruction is translated to less than 64 bytes | 1253 | * Before first pass, make a rough estimation of addrs[] |
| 1254 | * each BPF instruction is translated to less than 64 bytes | ||
| 1223 | */ | 1255 | */ |
| 1224 | for (proglen = 0, i = 0; i < prog->len; i++) { | 1256 | for (proglen = 0, i = 0; i < prog->len; i++) { |
| 1225 | proglen += 64; | 1257 | proglen += 64; |
| @@ -1228,10 +1260,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) | |||
| 1228 | ctx.cleanup_addr = proglen; | 1260 | ctx.cleanup_addr = proglen; |
| 1229 | skip_init_addrs: | 1261 | skip_init_addrs: |
| 1230 | 1262 | ||
| 1231 | /* JITed image shrinks with every pass and the loop iterates | 1263 | /* |
| 1232 | * until the image stops shrinking. Very large bpf programs | 1264 | * JITed image shrinks with every pass and the loop iterates |
| 1265 | * until the image stops shrinking. Very large BPF programs | ||
| 1233 | * may converge on the last pass. In such case do one more | 1266 | * may converge on the last pass. In such case do one more |
| 1234 | * pass to emit the final image | 1267 | * pass to emit the final image. |
| 1235 | */ | 1268 | */ |
| 1236 | for (pass = 0; pass < 20 || image; pass++) { | 1269 | for (pass = 0; pass < 20 || image; pass++) { |
| 1237 | proglen = do_jit(prog, addrs, image, oldproglen, &ctx); | 1270 | proglen = do_jit(prog, addrs, image, oldproglen, &ctx); |
