summaryrefslogtreecommitdiffstats
path: root/drivers/irqchip/irq-gic-v3-its.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-11-13 20:33:11 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2017-11-13 20:33:11 -0500
commit670310dfbae0eefe7318ff6a61e29e67a7a7bbce (patch)
treeeb3ce3aa3e6786a64fec93d410bb6f0b9a56be77 /drivers/irqchip/irq-gic-v3-its.c
parent43ff2f4db9d0f76452b77cfa645f02b471143b24 (diff)
parentffc661c99f621152d5fdcf53f9df0d48c326318b (diff)
Merge branch 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull irq core updates from Thomas Gleixner: "A rather large update for the interrupt core code and the irq chip drivers: - Add a new bitmap matrix allocator and supporting changes, which is used to replace the x86 vector allocator which comes with separate pull request. This allows to replace the convoluted nested loop allocation function in x86 with a facility which supports the recently added property of managed interrupts proper and allows to switch to a best effort vector reservation scheme, which addresses problems with vector exhaustion. - A large update to the ARM GIC-V3-ITS driver adding support for range selectors. - New interrupt controllers: - Meson and Meson8 GPIO - BCM7271 L2 - Socionext EXIU If you expected that this will stop at some point, I have to disappoint you. There are new ones posted already. Sigh! - STM32 interrupt controller support for new platforms. - A pile of fixes, cleanups and updates to the MIPS GIC driver - The usual small fixes, cleanups and updates all over the place. Most visible one is to move the irq chip drivers Kconfig switches into a separate Kconfig menu" * 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (70 commits) genirq: Fix type of shifting literal 1 in __setup_irq() irqdomain: Drop pointless NULL check in virq_debug_show_one genirq/proc: Return proper error code when irq_set_affinity() fails irq/work: Use llist_for_each_entry_safe irqchip: mips-gic: Print warning if inherited GIC base is used irqchip/mips-gic: Add pr_fmt and reword pr_* messages irqchip/stm32: Move the wakeup on interrupt mask irqchip/stm32: Fix initial values irqchip/stm32: Add stm32h7 support dt-bindings/interrupt-controllers: Add compatible string for stm32h7 irqchip/stm32: Add multi-bank management irqchip/stm32: Select GENERIC_IRQ_CHIP irqchip/exiu: Add support for Socionext Synquacer EXIU controller dt-bindings: Add description of Socionext EXIU interrupt controller irqchip/gic-v3-its: Fix VPE activate callback return value irqchip: mips-gic: Make IPI bitmaps static irqchip: mips-gic: Share register writes in gic_set_type() irqchip: mips-gic: Remove gic_vpes variable irqchip: mips-gic: Use num_possible_cpus() to reserve IPIs irqchip: mips-gic: Configure EIC when CPUs come online ...
Diffstat (limited to 'drivers/irqchip/irq-gic-v3-its.c')
-rw-r--r--drivers/irqchip/irq-gic-v3-its.c390
1 files changed, 310 insertions, 80 deletions
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index e88395605e32..4039e64cd342 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -83,6 +83,8 @@ struct its_baser {
83 u32 psz; 83 u32 psz;
84}; 84};
85 85
86struct its_device;
87
86/* 88/*
87 * The ITS structure - contains most of the infrastructure, with the 89 * The ITS structure - contains most of the infrastructure, with the
88 * top-level MSI domain, the command queue, the collections, and the 90 * top-level MSI domain, the command queue, the collections, and the
@@ -97,12 +99,18 @@ struct its_node {
97 struct its_cmd_block *cmd_write; 99 struct its_cmd_block *cmd_write;
98 struct its_baser tables[GITS_BASER_NR_REGS]; 100 struct its_baser tables[GITS_BASER_NR_REGS];
99 struct its_collection *collections; 101 struct its_collection *collections;
102 struct fwnode_handle *fwnode_handle;
103 u64 (*get_msi_base)(struct its_device *its_dev);
100 struct list_head its_device_list; 104 struct list_head its_device_list;
101 u64 flags; 105 u64 flags;
106 unsigned long list_nr;
102 u32 ite_size; 107 u32 ite_size;
103 u32 device_ids; 108 u32 device_ids;
104 int numa_node; 109 int numa_node;
110 unsigned int msi_domain_flags;
111 u32 pre_its_base; /* for Socionext Synquacer */
105 bool is_v4; 112 bool is_v4;
113 int vlpi_redist_offset;
106}; 114};
107 115
108#define ITS_ITT_ALIGN SZ_256 116#define ITS_ITT_ALIGN SZ_256
@@ -152,12 +160,6 @@ static DEFINE_SPINLOCK(its_lock);
152static struct rdists *gic_rdists; 160static struct rdists *gic_rdists;
153static struct irq_domain *its_parent; 161static struct irq_domain *its_parent;
154 162
155/*
156 * We have a maximum number of 16 ITSs in the whole system if we're
157 * using the ITSList mechanism
158 */
159#define ITS_LIST_MAX 16
160
161static unsigned long its_list_map; 163static unsigned long its_list_map;
162static u16 vmovp_seq_num; 164static u16 vmovp_seq_num;
163static DEFINE_RAW_SPINLOCK(vmovp_lock); 165static DEFINE_RAW_SPINLOCK(vmovp_lock);
@@ -272,10 +274,12 @@ struct its_cmd_block {
272#define ITS_CMD_QUEUE_SZ SZ_64K 274#define ITS_CMD_QUEUE_SZ SZ_64K
273#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) 275#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
274 276
275typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, 277typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
278 struct its_cmd_block *,
276 struct its_cmd_desc *); 279 struct its_cmd_desc *);
277 280
278typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_cmd_block *, 281typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
282 struct its_cmd_block *,
279 struct its_cmd_desc *); 283 struct its_cmd_desc *);
280 284
281static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 285static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
@@ -379,7 +383,8 @@ static inline void its_fixup_cmd(struct its_cmd_block *cmd)
379 cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); 383 cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
380} 384}
381 385
382static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, 386static struct its_collection *its_build_mapd_cmd(struct its_node *its,
387 struct its_cmd_block *cmd,
383 struct its_cmd_desc *desc) 388 struct its_cmd_desc *desc)
384{ 389{
385 unsigned long itt_addr; 390 unsigned long itt_addr;
@@ -399,7 +404,8 @@ static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd,
399 return NULL; 404 return NULL;
400} 405}
401 406
402static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, 407static struct its_collection *its_build_mapc_cmd(struct its_node *its,
408 struct its_cmd_block *cmd,
403 struct its_cmd_desc *desc) 409 struct its_cmd_desc *desc)
404{ 410{
405 its_encode_cmd(cmd, GITS_CMD_MAPC); 411 its_encode_cmd(cmd, GITS_CMD_MAPC);
@@ -412,7 +418,8 @@ static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd,
412 return desc->its_mapc_cmd.col; 418 return desc->its_mapc_cmd.col;
413} 419}
414 420
415static struct its_collection *its_build_mapti_cmd(struct its_cmd_block *cmd, 421static struct its_collection *its_build_mapti_cmd(struct its_node *its,
422 struct its_cmd_block *cmd,
416 struct its_cmd_desc *desc) 423 struct its_cmd_desc *desc)
417{ 424{
418 struct its_collection *col; 425 struct its_collection *col;
@@ -431,7 +438,8 @@ static struct its_collection *its_build_mapti_cmd(struct its_cmd_block *cmd,
431 return col; 438 return col;
432} 439}
433 440
434static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, 441static struct its_collection *its_build_movi_cmd(struct its_node *its,
442 struct its_cmd_block *cmd,
435 struct its_cmd_desc *desc) 443 struct its_cmd_desc *desc)
436{ 444{
437 struct its_collection *col; 445 struct its_collection *col;
@@ -449,7 +457,8 @@ static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd,
449 return col; 457 return col;
450} 458}
451 459
452static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, 460static struct its_collection *its_build_discard_cmd(struct its_node *its,
461 struct its_cmd_block *cmd,
453 struct its_cmd_desc *desc) 462 struct its_cmd_desc *desc)
454{ 463{
455 struct its_collection *col; 464 struct its_collection *col;
@@ -466,7 +475,8 @@ static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd,
466 return col; 475 return col;
467} 476}
468 477
469static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, 478static struct its_collection *its_build_inv_cmd(struct its_node *its,
479 struct its_cmd_block *cmd,
470 struct its_cmd_desc *desc) 480 struct its_cmd_desc *desc)
471{ 481{
472 struct its_collection *col; 482 struct its_collection *col;
@@ -483,7 +493,8 @@ static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd,
483 return col; 493 return col;
484} 494}
485 495
486static struct its_collection *its_build_int_cmd(struct its_cmd_block *cmd, 496static struct its_collection *its_build_int_cmd(struct its_node *its,
497 struct its_cmd_block *cmd,
487 struct its_cmd_desc *desc) 498 struct its_cmd_desc *desc)
488{ 499{
489 struct its_collection *col; 500 struct its_collection *col;
@@ -500,7 +511,8 @@ static struct its_collection *its_build_int_cmd(struct its_cmd_block *cmd,
500 return col; 511 return col;
501} 512}
502 513
503static struct its_collection *its_build_clear_cmd(struct its_cmd_block *cmd, 514static struct its_collection *its_build_clear_cmd(struct its_node *its,
515 struct its_cmd_block *cmd,
504 struct its_cmd_desc *desc) 516 struct its_cmd_desc *desc)
505{ 517{
506 struct its_collection *col; 518 struct its_collection *col;
@@ -517,7 +529,8 @@ static struct its_collection *its_build_clear_cmd(struct its_cmd_block *cmd,
517 return col; 529 return col;
518} 530}
519 531
520static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, 532static struct its_collection *its_build_invall_cmd(struct its_node *its,
533 struct its_cmd_block *cmd,
521 struct its_cmd_desc *desc) 534 struct its_cmd_desc *desc)
522{ 535{
523 its_encode_cmd(cmd, GITS_CMD_INVALL); 536 its_encode_cmd(cmd, GITS_CMD_INVALL);
@@ -528,7 +541,8 @@ static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd,
528 return NULL; 541 return NULL;
529} 542}
530 543
531static struct its_vpe *its_build_vinvall_cmd(struct its_cmd_block *cmd, 544static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
545 struct its_cmd_block *cmd,
532 struct its_cmd_desc *desc) 546 struct its_cmd_desc *desc)
533{ 547{
534 its_encode_cmd(cmd, GITS_CMD_VINVALL); 548 its_encode_cmd(cmd, GITS_CMD_VINVALL);
@@ -539,17 +553,20 @@ static struct its_vpe *its_build_vinvall_cmd(struct its_cmd_block *cmd,
539 return desc->its_vinvall_cmd.vpe; 553 return desc->its_vinvall_cmd.vpe;
540} 554}
541 555
542static struct its_vpe *its_build_vmapp_cmd(struct its_cmd_block *cmd, 556static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
557 struct its_cmd_block *cmd,
543 struct its_cmd_desc *desc) 558 struct its_cmd_desc *desc)
544{ 559{
545 unsigned long vpt_addr; 560 unsigned long vpt_addr;
561 u64 target;
546 562
547 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); 563 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
564 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
548 565
549 its_encode_cmd(cmd, GITS_CMD_VMAPP); 566 its_encode_cmd(cmd, GITS_CMD_VMAPP);
550 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); 567 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
551 its_encode_valid(cmd, desc->its_vmapp_cmd.valid); 568 its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
552 its_encode_target(cmd, desc->its_vmapp_cmd.col->target_address); 569 its_encode_target(cmd, target);
553 its_encode_vpt_addr(cmd, vpt_addr); 570 its_encode_vpt_addr(cmd, vpt_addr);
554 its_encode_vpt_size(cmd, LPI_NRBITS - 1); 571 its_encode_vpt_size(cmd, LPI_NRBITS - 1);
555 572
@@ -558,7 +575,8 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_cmd_block *cmd,
558 return desc->its_vmapp_cmd.vpe; 575 return desc->its_vmapp_cmd.vpe;
559} 576}
560 577
561static struct its_vpe *its_build_vmapti_cmd(struct its_cmd_block *cmd, 578static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
579 struct its_cmd_block *cmd,
562 struct its_cmd_desc *desc) 580 struct its_cmd_desc *desc)
563{ 581{
564 u32 db; 582 u32 db;
@@ -580,7 +598,8 @@ static struct its_vpe *its_build_vmapti_cmd(struct its_cmd_block *cmd,
580 return desc->its_vmapti_cmd.vpe; 598 return desc->its_vmapti_cmd.vpe;
581} 599}
582 600
583static struct its_vpe *its_build_vmovi_cmd(struct its_cmd_block *cmd, 601static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
602 struct its_cmd_block *cmd,
584 struct its_cmd_desc *desc) 603 struct its_cmd_desc *desc)
585{ 604{
586 u32 db; 605 u32 db;
@@ -602,14 +621,18 @@ static struct its_vpe *its_build_vmovi_cmd(struct its_cmd_block *cmd,
602 return desc->its_vmovi_cmd.vpe; 621 return desc->its_vmovi_cmd.vpe;
603} 622}
604 623
605static struct its_vpe *its_build_vmovp_cmd(struct its_cmd_block *cmd, 624static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
625 struct its_cmd_block *cmd,
606 struct its_cmd_desc *desc) 626 struct its_cmd_desc *desc)
607{ 627{
628 u64 target;
629
630 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
608 its_encode_cmd(cmd, GITS_CMD_VMOVP); 631 its_encode_cmd(cmd, GITS_CMD_VMOVP);
609 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); 632 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
610 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); 633 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
611 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); 634 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
612 its_encode_target(cmd, desc->its_vmovp_cmd.col->target_address); 635 its_encode_target(cmd, target);
613 636
614 its_fixup_cmd(cmd); 637 its_fixup_cmd(cmd);
615 638
@@ -688,9 +711,9 @@ static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
688 dsb(ishst); 711 dsb(ishst);
689} 712}
690 713
691static void its_wait_for_range_completion(struct its_node *its, 714static int its_wait_for_range_completion(struct its_node *its,
692 struct its_cmd_block *from, 715 struct its_cmd_block *from,
693 struct its_cmd_block *to) 716 struct its_cmd_block *to)
694{ 717{
695 u64 rd_idx, from_idx, to_idx; 718 u64 rd_idx, from_idx, to_idx;
696 u32 count = 1000000; /* 1s! */ 719 u32 count = 1000000; /* 1s! */
@@ -711,12 +734,15 @@ static void its_wait_for_range_completion(struct its_node *its,
711 734
712 count--; 735 count--;
713 if (!count) { 736 if (!count) {
714 pr_err_ratelimited("ITS queue timeout\n"); 737 pr_err_ratelimited("ITS queue timeout (%llu %llu %llu)\n",
715 return; 738 from_idx, to_idx, rd_idx);
739 return -1;
716 } 740 }
717 cpu_relax(); 741 cpu_relax();
718 udelay(1); 742 udelay(1);
719 } 743 }
744
745 return 0;
720} 746}
721 747
722/* Warning, macro hell follows */ 748/* Warning, macro hell follows */
@@ -736,7 +762,7 @@ void name(struct its_node *its, \
736 raw_spin_unlock_irqrestore(&its->lock, flags); \ 762 raw_spin_unlock_irqrestore(&its->lock, flags); \
737 return; \ 763 return; \
738 } \ 764 } \
739 sync_obj = builder(cmd, desc); \ 765 sync_obj = builder(its, cmd, desc); \
740 its_flush_cmd(its, cmd); \ 766 its_flush_cmd(its, cmd); \
741 \ 767 \
742 if (sync_obj) { \ 768 if (sync_obj) { \
@@ -744,7 +770,7 @@ void name(struct its_node *its, \
744 if (!sync_cmd) \ 770 if (!sync_cmd) \
745 goto post; \ 771 goto post; \
746 \ 772 \
747 buildfn(sync_cmd, sync_obj); \ 773 buildfn(its, sync_cmd, sync_obj); \
748 its_flush_cmd(its, sync_cmd); \ 774 its_flush_cmd(its, sync_cmd); \
749 } \ 775 } \
750 \ 776 \
@@ -752,10 +778,12 @@ post: \
752 next_cmd = its_post_commands(its); \ 778 next_cmd = its_post_commands(its); \
753 raw_spin_unlock_irqrestore(&its->lock, flags); \ 779 raw_spin_unlock_irqrestore(&its->lock, flags); \
754 \ 780 \
755 its_wait_for_range_completion(its, cmd, next_cmd); \ 781 if (its_wait_for_range_completion(its, cmd, next_cmd)) \
782 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
756} 783}
757 784
758static void its_build_sync_cmd(struct its_cmd_block *sync_cmd, 785static void its_build_sync_cmd(struct its_node *its,
786 struct its_cmd_block *sync_cmd,
759 struct its_collection *sync_col) 787 struct its_collection *sync_col)
760{ 788{
761 its_encode_cmd(sync_cmd, GITS_CMD_SYNC); 789 its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
@@ -767,7 +795,8 @@ static void its_build_sync_cmd(struct its_cmd_block *sync_cmd,
767static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t, 795static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
768 struct its_collection, its_build_sync_cmd) 796 struct its_collection, its_build_sync_cmd)
769 797
770static void its_build_vsync_cmd(struct its_cmd_block *sync_cmd, 798static void its_build_vsync_cmd(struct its_node *its,
799 struct its_cmd_block *sync_cmd,
771 struct its_vpe *sync_vpe) 800 struct its_vpe *sync_vpe)
772{ 801{
773 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC); 802 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
@@ -899,21 +928,16 @@ static void its_send_vmovi(struct its_device *dev, u32 id)
899 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); 928 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
900} 929}
901 930
902static void its_send_vmapp(struct its_vpe *vpe, bool valid) 931static void its_send_vmapp(struct its_node *its,
932 struct its_vpe *vpe, bool valid)
903{ 933{
904 struct its_cmd_desc desc; 934 struct its_cmd_desc desc;
905 struct its_node *its;
906 935
907 desc.its_vmapp_cmd.vpe = vpe; 936 desc.its_vmapp_cmd.vpe = vpe;
908 desc.its_vmapp_cmd.valid = valid; 937 desc.its_vmapp_cmd.valid = valid;
938 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
909 939
910 list_for_each_entry(its, &its_nodes, entry) { 940 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
911 if (!its->is_v4)
912 continue;
913
914 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
915 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
916 }
917} 941}
918 942
919static void its_send_vmovp(struct its_vpe *vpe) 943static void its_send_vmovp(struct its_vpe *vpe)
@@ -951,6 +975,9 @@ static void its_send_vmovp(struct its_vpe *vpe)
951 if (!its->is_v4) 975 if (!its->is_v4)
952 continue; 976 continue;
953 977
978 if (!vpe->its_vm->vlpi_count[its->list_nr])
979 continue;
980
954 desc.its_vmovp_cmd.col = &its->collections[col_id]; 981 desc.its_vmovp_cmd.col = &its->collections[col_id];
955 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 982 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
956 } 983 }
@@ -958,18 +985,12 @@ static void its_send_vmovp(struct its_vpe *vpe)
958 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 985 raw_spin_unlock_irqrestore(&vmovp_lock, flags);
959} 986}
960 987
961static void its_send_vinvall(struct its_vpe *vpe) 988static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
962{ 989{
963 struct its_cmd_desc desc; 990 struct its_cmd_desc desc;
964 struct its_node *its;
965 991
966 desc.its_vinvall_cmd.vpe = vpe; 992 desc.its_vinvall_cmd.vpe = vpe;
967 993 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
968 list_for_each_entry(its, &its_nodes, entry) {
969 if (!its->is_v4)
970 continue;
971 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
972 }
973} 994}
974 995
975/* 996/*
@@ -991,9 +1012,15 @@ static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
991 if (irqd_is_forwarded_to_vcpu(d)) { 1012 if (irqd_is_forwarded_to_vcpu(d)) {
992 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1013 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
993 u32 event = its_get_event_id(d); 1014 u32 event = its_get_event_id(d);
1015 struct its_vlpi_map *map;
994 1016
995 prop_page = its_dev->event_map.vm->vprop_page; 1017 prop_page = its_dev->event_map.vm->vprop_page;
996 hwirq = its_dev->event_map.vlpi_maps[event].vintid; 1018 map = &its_dev->event_map.vlpi_maps[event];
1019 hwirq = map->vintid;
1020
1021 /* Remember the updated property */
1022 map->properties &= ~clr;
1023 map->properties |= set | LPI_PROP_GROUP1;
997 } else { 1024 } else {
998 prop_page = gic_rdists->prop_page; 1025 prop_page = gic_rdists->prop_page;
999 hwirq = d->hwirq; 1026 hwirq = d->hwirq;
@@ -1099,6 +1126,13 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1099 return IRQ_SET_MASK_OK_DONE; 1126 return IRQ_SET_MASK_OK_DONE;
1100} 1127}
1101 1128
1129static u64 its_irq_get_msi_base(struct its_device *its_dev)
1130{
1131 struct its_node *its = its_dev->its;
1132
1133 return its->phys_base + GITS_TRANSLATER;
1134}
1135
1102static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) 1136static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1103{ 1137{
1104 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1138 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
@@ -1106,7 +1140,7 @@ static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1106 u64 addr; 1140 u64 addr;
1107 1141
1108 its = its_dev->its; 1142 its = its_dev->its;
1109 addr = its->phys_base + GITS_TRANSLATER; 1143 addr = its->get_msi_base(its_dev);
1110 1144
1111 msg->address_lo = lower_32_bits(addr); 1145 msg->address_lo = lower_32_bits(addr);
1112 msg->address_hi = upper_32_bits(addr); 1146 msg->address_hi = upper_32_bits(addr);
@@ -1133,6 +1167,60 @@ static int its_irq_set_irqchip_state(struct irq_data *d,
1133 return 0; 1167 return 0;
1134} 1168}
1135 1169
1170static void its_map_vm(struct its_node *its, struct its_vm *vm)
1171{
1172 unsigned long flags;
1173
1174 /* Not using the ITS list? Everything is always mapped. */
1175 if (!its_list_map)
1176 return;
1177
1178 raw_spin_lock_irqsave(&vmovp_lock, flags);
1179
1180 /*
1181 * If the VM wasn't mapped yet, iterate over the vpes and get
1182 * them mapped now.
1183 */
1184 vm->vlpi_count[its->list_nr]++;
1185
1186 if (vm->vlpi_count[its->list_nr] == 1) {
1187 int i;
1188
1189 for (i = 0; i < vm->nr_vpes; i++) {
1190 struct its_vpe *vpe = vm->vpes[i];
1191 struct irq_data *d = irq_get_irq_data(vpe->irq);
1192
1193 /* Map the VPE to the first possible CPU */
1194 vpe->col_idx = cpumask_first(cpu_online_mask);
1195 its_send_vmapp(its, vpe, true);
1196 its_send_vinvall(its, vpe);
1197 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1198 }
1199 }
1200
1201 raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1202}
1203
1204static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1205{
1206 unsigned long flags;
1207
1208 /* Not using the ITS list? Everything is always mapped. */
1209 if (!its_list_map)
1210 return;
1211
1212 raw_spin_lock_irqsave(&vmovp_lock, flags);
1213
1214 if (!--vm->vlpi_count[its->list_nr]) {
1215 int i;
1216
1217 for (i = 0; i < vm->nr_vpes; i++)
1218 its_send_vmapp(its, vm->vpes[i], false);
1219 }
1220
1221 raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1222}
1223
1136static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) 1224static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1137{ 1225{
1138 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1226 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
@@ -1168,12 +1256,23 @@ static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1168 /* Already mapped, move it around */ 1256 /* Already mapped, move it around */
1169 its_send_vmovi(its_dev, event); 1257 its_send_vmovi(its_dev, event);
1170 } else { 1258 } else {
1259 /* Ensure all the VPEs are mapped on this ITS */
1260 its_map_vm(its_dev->its, info->map->vm);
1261
1262 /*
1263 * Flag the interrupt as forwarded so that we can
1264 * start poking the virtual property table.
1265 */
1266 irqd_set_forwarded_to_vcpu(d);
1267
1268 /* Write out the property to the prop table */
1269 lpi_write_config(d, 0xff, info->map->properties);
1270
1171 /* Drop the physical mapping */ 1271 /* Drop the physical mapping */
1172 its_send_discard(its_dev, event); 1272 its_send_discard(its_dev, event);
1173 1273
1174 /* and install the virtual one */ 1274 /* and install the virtual one */
1175 its_send_vmapti(its_dev, event); 1275 its_send_vmapti(its_dev, event);
1176 irqd_set_forwarded_to_vcpu(d);
1177 1276
1178 /* Increment the number of VLPIs */ 1277 /* Increment the number of VLPIs */
1179 its_dev->event_map.nr_vlpis++; 1278 its_dev->event_map.nr_vlpis++;
@@ -1229,6 +1328,9 @@ static int its_vlpi_unmap(struct irq_data *d)
1229 LPI_PROP_ENABLED | 1328 LPI_PROP_ENABLED |
1230 LPI_PROP_GROUP1)); 1329 LPI_PROP_GROUP1));
1231 1330
1331 /* Potentially unmap the VM from this ITS */
1332 its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1333
1232 /* 1334 /*
1233 * Drop the refcount and make the device available again if 1335 * Drop the refcount and make the device available again if
1234 * this was the last VLPI. 1336 * this was the last VLPI.
@@ -1669,23 +1771,14 @@ static void its_free_tables(struct its_node *its)
1669 1771
1670static int its_alloc_tables(struct its_node *its) 1772static int its_alloc_tables(struct its_node *its)
1671{ 1773{
1672 u64 typer = gic_read_typer(its->base + GITS_TYPER);
1673 u32 ids = GITS_TYPER_DEVBITS(typer);
1674 u64 shr = GITS_BASER_InnerShareable; 1774 u64 shr = GITS_BASER_InnerShareable;
1675 u64 cache = GITS_BASER_RaWaWb; 1775 u64 cache = GITS_BASER_RaWaWb;
1676 u32 psz = SZ_64K; 1776 u32 psz = SZ_64K;
1677 int err, i; 1777 int err, i;
1678 1778
1679 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) { 1779 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
1680 /* 1780 /* erratum 24313: ignore memory access type */
1681 * erratum 22375: only alloc 8MB table size 1781 cache = GITS_BASER_nCnB;
1682 * erratum 24313: ignore memory access type
1683 */
1684 cache = GITS_BASER_nCnB;
1685 ids = 0x14; /* 20 bits, 8MB */
1686 }
1687
1688 its->device_ids = ids;
1689 1782
1690 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 1783 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
1691 struct its_baser *baser = its->tables + i; 1784 struct its_baser *baser = its->tables + i;
@@ -2209,8 +2302,8 @@ static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
2209 return 0; 2302 return 0;
2210} 2303}
2211 2304
2212static void its_irq_domain_activate(struct irq_domain *domain, 2305static int its_irq_domain_activate(struct irq_domain *domain,
2213 struct irq_data *d) 2306 struct irq_data *d, bool early)
2214{ 2307{
2215 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 2308 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2216 u32 event = its_get_event_id(d); 2309 u32 event = its_get_event_id(d);
@@ -2228,6 +2321,7 @@ static void its_irq_domain_activate(struct irq_domain *domain,
2228 2321
2229 /* Map the GIC IRQ and event to the device */ 2322 /* Map the GIC IRQ and event to the device */
2230 its_send_mapti(its_dev, d->hwirq, event); 2323 its_send_mapti(its_dev, d->hwirq, event);
2324 return 0;
2231} 2325}
2232 2326
2233static void its_irq_domain_deactivate(struct irq_domain *domain, 2327static void its_irq_domain_deactivate(struct irq_domain *domain,
@@ -2394,6 +2488,8 @@ static int its_vpe_set_affinity(struct irq_data *d,
2394 its_vpe_db_proxy_move(vpe, from, cpu); 2488 its_vpe_db_proxy_move(vpe, from, cpu);
2395 } 2489 }
2396 2490
2491 irq_data_update_effective_affinity(d, cpumask_of(cpu));
2492
2397 return IRQ_SET_MASK_OK_DONE; 2493 return IRQ_SET_MASK_OK_DONE;
2398} 2494}
2399 2495
@@ -2461,6 +2557,26 @@ static void its_vpe_deschedule(struct its_vpe *vpe)
2461 } 2557 }
2462} 2558}
2463 2559
2560static void its_vpe_invall(struct its_vpe *vpe)
2561{
2562 struct its_node *its;
2563
2564 list_for_each_entry(its, &its_nodes, entry) {
2565 if (!its->is_v4)
2566 continue;
2567
2568 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
2569 continue;
2570
2571 /*
2572 * Sending a VINVALL to a single ITS is enough, as all
2573 * we need is to reach the redistributors.
2574 */
2575 its_send_vinvall(its, vpe);
2576 return;
2577 }
2578}
2579
2464static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 2580static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
2465{ 2581{
2466 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2582 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
@@ -2476,7 +2592,7 @@ static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
2476 return 0; 2592 return 0;
2477 2593
2478 case INVALL_VPE: 2594 case INVALL_VPE:
2479 its_send_vinvall(vpe); 2595 its_vpe_invall(vpe);
2480 return 0; 2596 return 0;
2481 2597
2482 default: 2598 default:
@@ -2701,23 +2817,51 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq
2701 return err; 2817 return err;
2702} 2818}
2703 2819
2704static void its_vpe_irq_domain_activate(struct irq_domain *domain, 2820static int its_vpe_irq_domain_activate(struct irq_domain *domain,
2705 struct irq_data *d) 2821 struct irq_data *d, bool early)
2706{ 2822{
2707 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2823 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2824 struct its_node *its;
2825
2826 /* If we use the list map, we issue VMAPP on demand... */
2827 if (its_list_map)
2828 return 0;
2708 2829
2709 /* Map the VPE to the first possible CPU */ 2830 /* Map the VPE to the first possible CPU */
2710 vpe->col_idx = cpumask_first(cpu_online_mask); 2831 vpe->col_idx = cpumask_first(cpu_online_mask);
2711 its_send_vmapp(vpe, true); 2832
2712 its_send_vinvall(vpe); 2833 list_for_each_entry(its, &its_nodes, entry) {
2834 if (!its->is_v4)
2835 continue;
2836
2837 its_send_vmapp(its, vpe, true);
2838 its_send_vinvall(its, vpe);
2839 }
2840
2841 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
2842
2843 return 0;
2713} 2844}
2714 2845
2715static void its_vpe_irq_domain_deactivate(struct irq_domain *domain, 2846static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
2716 struct irq_data *d) 2847 struct irq_data *d)
2717{ 2848{
2718 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 2849 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2850 struct its_node *its;
2851
2852 /*
2853 * If we use the list map, we unmap the VPE once no VLPIs are
2854 * associated with the VM.
2855 */
2856 if (its_list_map)
2857 return;
2719 2858
2720 its_send_vmapp(vpe, false); 2859 list_for_each_entry(its, &its_nodes, entry) {
2860 if (!its->is_v4)
2861 continue;
2862
2863 its_send_vmapp(its, vpe, false);
2864 }
2721} 2865}
2722 2866
2723static const struct irq_domain_ops its_vpe_domain_ops = { 2867static const struct irq_domain_ops its_vpe_domain_ops = {
@@ -2760,26 +2904,85 @@ static int its_force_quiescent(void __iomem *base)
2760 } 2904 }
2761} 2905}
2762 2906
2763static void __maybe_unused its_enable_quirk_cavium_22375(void *data) 2907static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
2764{ 2908{
2765 struct its_node *its = data; 2909 struct its_node *its = data;
2766 2910
2911 /* erratum 22375: only alloc 8MB table size */
2912 its->device_ids = 0x14; /* 20 bits, 8MB */
2767 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; 2913 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
2914
2915 return true;
2768} 2916}
2769 2917
2770static void __maybe_unused its_enable_quirk_cavium_23144(void *data) 2918static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
2771{ 2919{
2772 struct its_node *its = data; 2920 struct its_node *its = data;
2773 2921
2774 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; 2922 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
2923
2924 return true;
2775} 2925}
2776 2926
2777static void __maybe_unused its_enable_quirk_qdf2400_e0065(void *data) 2927static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
2778{ 2928{
2779 struct its_node *its = data; 2929 struct its_node *its = data;
2780 2930
2781 /* On QDF2400, the size of the ITE is 16Bytes */ 2931 /* On QDF2400, the size of the ITE is 16Bytes */
2782 its->ite_size = 16; 2932 its->ite_size = 16;
2933
2934 return true;
2935}
2936
2937static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
2938{
2939 struct its_node *its = its_dev->its;
2940
2941 /*
2942 * The Socionext Synquacer SoC has a so-called 'pre-ITS',
2943 * which maps 32-bit writes targeted at a separate window of
2944 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
2945 * with device ID taken from bits [device_id_bits + 1:2] of
2946 * the window offset.
2947 */
2948 return its->pre_its_base + (its_dev->device_id << 2);
2949}
2950
2951static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
2952{
2953 struct its_node *its = data;
2954 u32 pre_its_window[2];
2955 u32 ids;
2956
2957 if (!fwnode_property_read_u32_array(its->fwnode_handle,
2958 "socionext,synquacer-pre-its",
2959 pre_its_window,
2960 ARRAY_SIZE(pre_its_window))) {
2961
2962 its->pre_its_base = pre_its_window[0];
2963 its->get_msi_base = its_irq_get_msi_base_pre_its;
2964
2965 ids = ilog2(pre_its_window[1]) - 2;
2966 if (its->device_ids > ids)
2967 its->device_ids = ids;
2968
2969 /* the pre-ITS breaks isolation, so disable MSI remapping */
2970 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
2971 return true;
2972 }
2973 return false;
2974}
2975
2976static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
2977{
2978 struct its_node *its = data;
2979
2980 /*
2981 * Hip07 insists on using the wrong address for the VLPI
2982 * page. Trick it into doing the right thing...
2983 */
2984 its->vlpi_redist_offset = SZ_128K;
2985 return true;
2783} 2986}
2784 2987
2785static const struct gic_quirk its_quirks[] = { 2988static const struct gic_quirk its_quirks[] = {
@@ -2807,6 +3010,27 @@ static const struct gic_quirk its_quirks[] = {
2807 .init = its_enable_quirk_qdf2400_e0065, 3010 .init = its_enable_quirk_qdf2400_e0065,
2808 }, 3011 },
2809#endif 3012#endif
3013#ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
3014 {
3015 /*
3016 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
3017 * implementation, but with a 'pre-ITS' added that requires
3018 * special handling in software.
3019 */
3020 .desc = "ITS: Socionext Synquacer pre-ITS",
3021 .iidr = 0x0001143b,
3022 .mask = 0xffffffff,
3023 .init = its_enable_quirk_socionext_synquacer,
3024 },
3025#endif
3026#ifdef CONFIG_HISILICON_ERRATUM_161600802
3027 {
3028 .desc = "ITS: Hip07 erratum 161600802",
3029 .iidr = 0x00000004,
3030 .mask = 0xffffffff,
3031 .init = its_enable_quirk_hip07_161600802,
3032 },
3033#endif
2810 { 3034 {
2811 } 3035 }
2812}; 3036};
@@ -2835,7 +3059,7 @@ static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
2835 3059
2836 inner_domain->parent = its_parent; 3060 inner_domain->parent = its_parent;
2837 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); 3061 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
2838 inner_domain->flags |= IRQ_DOMAIN_FLAG_MSI_REMAP; 3062 inner_domain->flags |= its->msi_domain_flags;
2839 info->ops = &its_msi_domain_ops; 3063 info->ops = &its_msi_domain_ops;
2840 info->data = its; 3064 info->data = its;
2841 inner_domain->host_data = info; 3065 inner_domain->host_data = info;
@@ -2896,8 +3120,8 @@ static int __init its_compute_its_list_map(struct resource *res,
2896 * locking. Should this change, we should address 3120 * locking. Should this change, we should address
2897 * this. 3121 * this.
2898 */ 3122 */
2899 its_number = find_first_zero_bit(&its_list_map, ITS_LIST_MAX); 3123 its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
2900 if (its_number >= ITS_LIST_MAX) { 3124 if (its_number >= GICv4_ITS_LIST_MAX) {
2901 pr_err("ITS@%pa: No ITSList entry available!\n", 3125 pr_err("ITS@%pa: No ITSList entry available!\n",
2902 &res->start); 3126 &res->start);
2903 return -EINVAL; 3127 return -EINVAL;
@@ -2965,6 +3189,7 @@ static int __init its_probe_one(struct resource *res,
2965 its->base = its_base; 3189 its->base = its_base;
2966 its->phys_base = res->start; 3190 its->phys_base = res->start;
2967 its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer); 3191 its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer);
3192 its->device_ids = GITS_TYPER_DEVBITS(typer);
2968 its->is_v4 = !!(typer & GITS_TYPER_VLPIS); 3193 its->is_v4 = !!(typer & GITS_TYPER_VLPIS);
2969 if (its->is_v4) { 3194 if (its->is_v4) {
2970 if (!(typer & GITS_TYPER_VMOVP)) { 3195 if (!(typer & GITS_TYPER_VMOVP)) {
@@ -2972,6 +3197,8 @@ static int __init its_probe_one(struct resource *res,
2972 if (err < 0) 3197 if (err < 0)
2973 goto out_free_its; 3198 goto out_free_its;
2974 3199
3200 its->list_nr = err;
3201
2975 pr_info("ITS@%pa: Using ITS number %d\n", 3202 pr_info("ITS@%pa: Using ITS number %d\n",
2976 &res->start, err); 3203 &res->start, err);
2977 } else { 3204 } else {
@@ -2988,6 +3215,9 @@ static int __init its_probe_one(struct resource *res,
2988 goto out_free_its; 3215 goto out_free_its;
2989 } 3216 }
2990 its->cmd_write = its->cmd_base; 3217 its->cmd_write = its->cmd_base;
3218 its->fwnode_handle = handle;
3219 its->get_msi_base = its_irq_get_msi_base;
3220 its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP;
2991 3221
2992 its_enable_quirks(its); 3222 its_enable_quirks(its);
2993 3223