aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net/s2io-regs.h
blob: 416669fd68c6bcbaf42b4ebc4962195713f1d459 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
/************************************************************************
 * regs.h: A Linux PCI-X Ethernet driver for Neterion 10GbE Server NIC
 * Copyright(c) 2002-2007 Neterion Inc.

 * This software may be used and distributed according to the terms of
 * the GNU General Public License (GPL), incorporated herein by reference.
 * Drivers based on or derived from this code fall under the GPL and must
 * retain the authorship, copyright and license notice.  This file is not
 * a complete program and may only be used when the entire operating
 * system is licensed under the GPL.
 * See the file COPYING in this distribution for more information.
 ************************************************************************/
#ifndef _REGS_H
#define _REGS_H

#define TBD 0

struct XENA_dev_config {
/* Convention: mHAL_XXX is mask, vHAL_XXX is value */

/* General Control-Status Registers */
	u64 general_int_status;
#define GEN_INTR_TXPIC             s2BIT(0)
#define GEN_INTR_TXDMA             s2BIT(1)
#define GEN_INTR_TXMAC             s2BIT(2)
#define GEN_INTR_TXXGXS            s2BIT(3)
#define GEN_INTR_TXTRAFFIC         s2BIT(8)
#define GEN_INTR_RXPIC             s2BIT(32)
#define GEN_INTR_RXDMA             s2BIT(33)
#define GEN_INTR_RXMAC             s2BIT(34)
#define GEN_INTR_MC                s2BIT(35)
#define GEN_INTR_RXXGXS            s2BIT(36)
#define GEN_INTR_RXTRAFFIC         s2BIT(40)
#define GEN_ERROR_INTR             GEN_INTR_TXPIC | GEN_INTR_RXPIC | \
                                   GEN_INTR_TXDMA | GEN_INTR_RXDMA | \
                                   GEN_INTR_TXMAC | GEN_INTR_RXMAC | \
                                   GEN_INTR_TXXGXS| GEN_INTR_RXXGXS| \
                                   GEN_INTR_MC

	u64 general_int_mask;

	u8 unused0[0x100 - 0x10];

	u64 sw_reset;
/* XGXS must be removed from reset only once. */
#define SW_RESET_XENA              vBIT(0xA5,0,8)
#define SW_RESET_FLASH             vBIT(0xA5,8,8)
#define SW_RESET_EOI               vBIT(0xA5,16,8)
#define SW_RESET_ALL               (SW_RESET_XENA     |   \
                                    SW_RESET_FLASH    |   \
                                    SW_RESET_EOI)
/* The SW_RESET register must read this value after a successful reset. */
#define	SW_RESET_RAW_VAL			0xA5000000


	u64 adapter_status;
#define ADAPTER_STATUS_TDMA_READY          s2BIT(0)
#define ADAPTER_STATUS_RDMA_READY          s2BIT(1)
#define ADAPTER_STATUS_PFC_READY           s2BIT(2)
#define ADAPTER_STATUS_TMAC_BUF_EMPTY      s2BIT(3)
#define ADAPTER_STATUS_PIC_QUIESCENT       s2BIT(5)
#define ADAPTER_STATUS_RMAC_REMOTE_FAULT   s2BIT(6)
#define ADAPTER_STATUS_RMAC_LOCAL_FAULT    s2BIT(7)
#define ADAPTER_STATUS_RMAC_PCC_IDLE       vBIT(0xFF,8,8)
#define ADAPTER_STATUS_RMAC_PCC_FOUR_IDLE  vBIT(0x0F,8,8)
#define ADAPTER_STATUS_RC_PRC_QUIESCENT    vBIT(0xFF,16,8)
#define ADAPTER_STATUS_MC_DRAM_READY       s2BIT(24)
#define ADAPTER_STATUS_MC_QUEUES_READY     s2BIT(25)
#define ADAPTER_STATUS_RIC_RUNNING         s2BIT(26)
#define ADAPTER_STATUS_M_PLL_LOCK          s2BIT(30)
#define ADAPTER_STATUS_P_PLL_LOCK          s2BIT(31)

	u64 adapter_control;
#define ADAPTER_CNTL_EN                    s2BIT(7)
#define ADAPTER_EOI_TX_ON                  s2BIT(15)
#define ADAPTER_LED_ON                     s2BIT(23)
#define ADAPTER_UDPI(val)                  vBIT(val,36,4)
#define ADAPTER_WAIT_INT                   s2BIT(48)
#define ADAPTER_ECC_EN                     s2BIT(55)

	u64 serr_source;
#define SERR_SOURCE_PIC			s2BIT(0)
#define SERR_SOURCE_TXDMA		s2BIT(1)
#define SERR_SOURCE_RXDMA		s2BIT(2)
#define SERR_SOURCE_MAC                 s2BIT(3)
#define SERR_SOURCE_MC                  s2BIT(4)
#define SERR_SOURCE_XGXS                s2BIT(5)
#define	SERR_SOURCE_ANY			(SERR_SOURCE_PIC	| \
					SERR_SOURCE_TXDMA	| \
					SERR_SOURCE_RXDMA	| \
					SERR_SOURCE_MAC		| \
					SERR_SOURCE_MC		| \
					SERR_SOURCE_XGXS)

	u64 pci_mode;
#define	GET_PCI_MODE(val)		((val & vBIT(0xF, 0, 4)) >> 60)
#define	PCI_MODE_PCI_33			0
#define	PCI_MODE_PCI_66			0x1
#define	PCI_MODE_PCIX_M1_66		0x2
#define	PCI_MODE_PCIX_M1_100		0x3
#define	PCI_MODE_PCIX_M1_133		0x4
#define	PCI_MODE_PCIX_M2_66		0x5
#define	PCI_MODE_PCIX_M2_100		0x6
#define	PCI_MODE_PCIX_M2_133		0x7
#define	PCI_MODE_UNSUPPORTED		s2BIT(0)
#define	PCI_MODE_32_BITS		s2BIT(8)
#define	PCI_MODE_UNKNOWN_MODE		s2BIT(9)

	u8 unused_0[0x800 - 0x128];

/* PCI-X Controller registers */
	u64 pic_int_status;
	u64 pic_int_mask;
#define PIC_INT_TX                     s2BIT(0)
#define PIC_INT_FLSH                   s2BIT(1)
#define PIC_INT_MDIO                   s2BIT(2)
#define PIC_INT_IIC                    s2BIT(3)
#define PIC_INT_GPIO                   s2BIT(4)
#define PIC_INT_RX                     s2BIT(32)

	u64 txpic_int_reg;
	u64 txpic_int_mask;
#define PCIX_INT_REG_ECC_SG_ERR                s2BIT(0)
#define PCIX_INT_REG_ECC_DB_ERR                s2BIT(1)
#define PCIX_INT_REG_FLASHR_R_FSM_ERR          s2BIT(8)
#define PCIX_INT_REG_FLASHR_W_FSM_ERR          s2BIT(9)
#define PCIX_INT_REG_INI_TX_FSM_SERR           s2BIT(10)
#define PCIX_INT_REG_INI_TXO_FSM_ERR           s2BIT(11)
#define PCIX_INT_REG_TRT_FSM_SERR              s2BIT(13)
#define PCIX_INT_REG_SRT_FSM_SERR              s2BIT(14)
#define PCIX_INT_REG_PIFR_FSM_SERR             s2BIT(15)
#define PCIX_INT_REG_WRC_TX_SEND_FSM_SERR      s2BIT(21)
#define PCIX_INT_REG_RRC_TX_REQ_FSM_SERR       s2BIT(23)
#define PCIX_INT_REG_INI_RX_FSM_SERR           s2BIT(48)
#define PCIX_INT_REG_RA_RX_FSM_SERR            s2BIT(50)
/*
#define PCIX_INT_REG_WRC_RX_SEND_FSM_SERR      s2BIT(52)
#define PCIX_INT_REG_RRC_RX_REQ_FSM_SERR       s2BIT(54)
#define PCIX_INT_REG_RRC_RX_SPLIT_FSM_SERR     s2BIT(58)
*/
	u64 txpic_alarms;
	u64 rxpic_int_reg;
	u64 rxpic_int_mask;
	u64 rxpic_alarms;

	u64 flsh_int_reg;
	u64 flsh_int_mask;
#define PIC_FLSH_INT_REG_CYCLE_FSM_ERR         s2BIT(63)
#define PIC_FLSH_INT_REG_ERR                   s2BIT(62)
	u64 flash_alarms;

	u64 mdio_int_reg;
	u64 mdio_int_mask;
#define MDIO_INT_REG_MDIO_BUS_ERR              s2BIT(0)
#define MDIO_INT_REG_DTX_BUS_ERR               s2BIT(8)
#define MDIO_INT_REG_LASI                      s2BIT(39)
	u64 mdio_alarms;

	u64 iic_int_reg;
	u64 iic_int_mask;
#define IIC_INT_REG_BUS_FSM_ERR                s2BIT(4)
#define IIC_INT_REG_BIT_FSM_ERR                s2BIT(5)
#define IIC_INT_REG_CYCLE_FSM_ERR              s2BIT(6)
#define IIC_INT_REG_REQ_FSM_ERR                s2BIT(7)
#define IIC_INT_REG_ACK_ERR                    s2BIT(8)
	u64 iic_alarms;

	u8 unused4[0x08];

	u64 gpio_int_reg;
#define GPIO_INT_REG_DP_ERR_INT                s2BIT(0)
#define GPIO_INT_REG_LINK_DOWN                 s2BIT(1)
#define GPIO_INT_REG_LINK_UP                   s2BIT(2)
	u64 gpio_int_mask;
#define GPIO_INT_MASK_LINK_DOWN                s2BIT(1)
#define GPIO_INT_MASK_LINK_UP                  s2BIT(2)
	u64 gpio_alarms;

	u8 unused5[0x38];

	u64 tx_traffic_int;
#define TX_TRAFFIC_INT_n(n)                    s2BIT(n)
	u64 tx_traffic_mask;

	u64 rx_traffic_int;
#define RX_TRAFFIC_INT_n(n)                    s2BIT(n)
	u64 rx_traffic_mask;

/* PIC Control registers */
	u64 pic_control;
#define PIC_CNTL_RX_ALARM_MAP_1                s2BIT(0)
#define PIC_CNTL_SHARED_SPLITS(n)              vBIT(n,11,5)

	u64 swapper_ctrl;
#define SWAPPER_CTRL_PIF_R_FE                  s2BIT(0)
#define SWAPPER_CTRL_PIF_R_SE                  s2BIT(1)
#define SWAPPER_CTRL_PIF_W_FE                  s2BIT(8)
#define SWAPPER_CTRL_PIF_W_SE                  s2BIT(9)
#define SWAPPER_CTRL_TXP_FE                    s2BIT(16)
#define SWAPPER_CTRL_TXP_SE                    s2BIT(17)
#define SWAPPER_CTRL_TXD_R_FE                  s2BIT(18)
#define SWAPPER_CTRL_TXD_R_SE                  s2BIT(19)
#define SWAPPER_CTRL_TXD_W_FE                  s2BIT(20)
#define SWAPPER_CTRL_TXD_W_SE                  s2BIT(21)
#define SWAPPER_CTRL_TXF_R_FE                  s2BIT(22)
#define SWAPPER_CTRL_TXF_R_SE                  s2BIT(23)
#define SWAPPER_CTRL_RXD_R_FE                  s2BIT(32)
#define SWAPPER_CTRL_RXD_R_SE                  s2BIT(33)
#define SWAPPER_CTRL_RXD_W_FE                  s2BIT(34)
#define SWAPPER_CTRL_RXD_W_SE                  s2BIT(35)
#define SWAPPER_CTRL_RXF_W_FE                  s2BIT(36)
#define SWAPPER_CTRL_RXF_W_SE                  s2BIT(37)
#define SWAPPER_CTRL_XMSI_FE                   s2BIT(40)
#define SWAPPER_CTRL_XMSI_SE                   s2BIT(41)
#define SWAPPER_CTRL_STATS_FE                  s2BIT(48)
#define SWAPPER_CTRL_STATS_SE                  s2BIT(49)

	u64 pif_rd_swapper_fb;
#define IF_RD_SWAPPER_FB                            0x0123456789ABCDEF

	u64 scheduled_int_ctrl;
#define SCHED_INT_CTRL_TIMER_EN                s2BIT(0)
#define SCHED_INT_CTRL_ONE_SHOT                s2BIT(1)
#define SCHED_INT_CTRL_INT2MSI(val)		vBIT(val,10,6)
#define SCHED_INT_PERIOD                       TBD

	u64 txreqtimeout;
#define TXREQTO_VAL(val)						vBIT(val,0,32)
#define TXREQTO_EN								s2BIT(63)

	u64 statsreqtimeout;
#define STATREQTO_VAL(n)                       TBD
#define STATREQTO_EN                           s2BIT(63)

	u64 read_retry_delay;
	u64 read_retry_acceleration;
	u64 write_retry_delay;
	u64 write_retry_acceleration;

	u64 xmsi_control;
	u64 xmsi_access;
	u64 xmsi_address;
	u64 xmsi_data;

	u64 rx_mat;
#define RX_MAT_SET(ring, msi)			vBIT(msi, (8 * ring), 8)

	u8 unused6[0x8];

	u64 tx_mat0_n[0x8];
#define TX_MAT_SET(fifo, msi)			vBIT(msi, (8 * fifo), 8)

	u64 xmsi_mask_reg;
	u64 stat_byte_cnt;
#define STAT_BC(n)                              vBIT(n,4,12)

	/* Automated statistics collection */
	u64 stat_cfg;
#define STAT_CFG_STAT_EN           s2BIT(0)
#define STAT_CFG_ONE_SHOT_EN       s2BIT(1)
#define STAT_CFG_STAT_NS_EN        s2BIT(8)
#define STAT_CFG_STAT_RO           s2BIT(9)
#define STAT_TRSF_PER(n)           TBD
#define	PER_SEC					   0x208d5
#define	SET_UPDT_PERIOD(n)		   vBIT((PER_SEC*n),32,32)
#define	SET_UPDT_CLICKS(val)		   vBIT(val, 32, 32)

	u64 stat_addr;

	/* General Configuration */
	u64 mdio_control;
#define MDIO_MMD_INDX_ADDR(val)		vBIT(val, 0, 16)
#define MDIO_MMD_DEV_ADDR(val)		vBIT(val, 19, 5)
#define MDIO_MMS_PRT_ADDR(val)		vBIT(val, 27, 5)
#define MDIO_CTRL_START_TRANS(val)	vBIT(val, 56, 4)
#define MDIO_OP(val)			vBIT(val, 60, 2)
#define MDIO_OP_ADDR_TRANS		0x0
#define MDIO_OP_WRITE_TRANS		0x1
#define MDIO_OP_READ_POST_INC_TRANS	0x2
#define MDIO_OP_READ_TRANS		0x3
#define MDIO_MDIO_DATA(val)		vBIT(val, 32, 16)

	u64 dtx_control;

	u64 i2c_control;
#define	I2C_CONTROL_DEV_ID(id)		vBIT(id,1,3)
#define	I2C_CONTROL_ADDR(addr)		vBIT(addr,5,11)
#define	I2C_CONTROL_BYTE_CNT(cnt)	vBIT(cnt,22,2)
#define	I2C_CONTROL_READ			s2BIT(24)
#define	I2C_CONTROL_NACK			s2BIT(25)
#define	I2C_CONTROL_CNTL_START		vBIT(0xE,28,4)
#define	I2C_CONTROL_CNTL_END(val)	(val & vBIT(0x1,28,4))
#define	I2C_CONTROL_GET_DATA(val)	(u32)(val & 0xFFFFFFFF)
#define	I2C_CONTROL_SET_DATA(val)	vBIT(val,32,32)

	u64 gpio_control;
#define GPIO_CTRL_GPIO_0		s2BIT(8)
	u64 misc_control;
#define FAULT_BEHAVIOUR			s2BIT(0)
#define EXT_REQ_EN			s2BIT(1)
#define MISC_LINK_STABILITY_PRD(val)   vBIT(val,29,3)

	u8 unused7_1[0x230 - 0x208];

	u64 pic_control2;
	u64 ini_dperr_ctrl;

	u64 wreq_split_mask;
#define	WREQ_SPLIT_MASK_SET_MASK(val)	vBIT(val, 52, 12)

	u8 unused7_2[0x800 - 0x248];

/* TxDMA registers */
	u64 txdma_int_status;
	u64 txdma_int_mask;
#define TXDMA_PFC_INT                  s2BIT(0)
#define TXDMA_TDA_INT                  s2BIT(1)
#define TXDMA_PCC_INT                  s2BIT(2)
#define TXDMA_TTI_INT                  s2BIT(3)
#define TXDMA_LSO_INT                  s2BIT(4)
#define TXDMA_TPA_INT                  s2BIT(5)
#define TXDMA_SM_INT                   s2BIT(6)
	u64 pfc_err_reg;
#define PFC_ECC_SG_ERR			s2BIT(7)
#define PFC_ECC_DB_ERR			s2BIT(15)
#define PFC_SM_ERR_ALARM		s2BIT(23)
#define PFC_MISC_0_ERR			s2BIT(31)
#define PFC_MISC_1_ERR			s2BIT(32)
#define PFC_PCIX_ERR			s2BIT(39)
	u64 pfc_err_mask;
	u64 pfc_err_alarm;

	u64 tda_err_reg;
#define TDA_Fn_ECC_SG_ERR		vBIT(0xff,0,8)
#define TDA_Fn_ECC_DB_ERR		vBIT(0xff,8,8)
#define TDA_SM0_ERR_ALARM		s2BIT(22)
#define TDA_SM1_ERR_ALARM		s2BIT(23)
#define TDA_PCIX_ERR			s2BIT(39)
	u64 tda_err_mask;
	u64 tda_err_alarm;

	u64 pcc_err_reg;
#define PCC_FB_ECC_SG_ERR		vBIT(0xFF,0,8)
#define PCC_TXB_ECC_SG_ERR		vBIT(0xFF,8,8)
#define PCC_FB_ECC_DB_ERR		vBIT(0xFF,16, 8)
#define PCC_TXB_ECC_DB_ERR		vBIT(0xff,24,8)
#define PCC_SM_ERR_ALARM		vBIT(0xff,32,8)
#define PCC_WR_ERR_ALARM		vBIT(0xff,40,8)
#define PCC_N_SERR			vBIT(0xff,48,8)
#define PCC_6_COF_OV_ERR		s2BIT(56)
#define PCC_7_COF_OV_ERR		s2BIT(57)
#define PCC_6_LSO_OV_ERR		s2BIT(58)
#define PCC_7_LSO_OV_ERR		s2BIT(59)
#define PCC_ENABLE_FOUR			vBIT(0x0F,0,8)
	u64 pcc_err_mask;
	u64 pcc_err_alarm;

	u64 tti_err_reg;
#define TTI_ECC_SG_ERR			s2BIT(7)
#define TTI_ECC_DB_ERR			s2BIT(15)
#define TTI_SM_ERR_ALARM		s2BIT(23)
	u64 tti_err_mask;
	u64 tti_err_alarm;

	u64 lso_err_reg;
#define LSO6_SEND_OFLOW			s2BIT(12)
#define LSO7_SEND_OFLOW			s2BIT(13)
#define LSO6_ABORT			s2BIT(14)
#define LSO7_ABORT			s2BIT(15)
#define LSO6_SM_ERR_ALARM		s2BIT(22)
#define LSO7_SM_ERR_ALARM		s2BIT(23)
	u64 lso_err_mask;
	u64 lso_err_alarm;

	u64 tpa_err_reg;
#define TPA_TX_FRM_DROP			s2BIT(7)
#define TPA_SM_ERR_ALARM		s2BIT(23)

	u64 tpa_err_mask;
	u64 tpa_err_alarm;

	u64 sm_err_reg;
#define SM_SM_ERR_ALARM			s2BIT(15)
	u64 sm_err_mask;
	u64 sm_err_alarm;

	u8 unused8[0x100 - 0xB8];

/* TxDMA arbiter */
	u64 tx_dma_wrap_stat;

/* Tx FIFO controller */
#define X_MAX_FIFOS                        8
#define X_FIFO_MAX_LEN                     0x1FFF	/*8191 */
	u64 tx_fifo_partition_0;
#define TX_FIFO_PARTITION_EN               s2BIT(0)
#define TX_FIFO_PARTITION_0_PRI(val)       vBIT(val,5,3)
#define TX_FIFO_PARTITION_0_LEN(val)       vBIT(val,19,13)
#define TX_FIFO_PARTITION_1_PRI(val)       vBIT(val,37,3)
#define TX_FIFO_PARTITION_1_LEN(val)       vBIT(val,51,13  )

	u64 tx_fifo_partition_1;
#define TX_FIFO_PARTITION_2_PRI(val)       vBIT(val,5,3)
#define TX_FIFO_PARTITION_2_LEN(val)       vBIT(val,19,13)
#define TX_FIFO_PARTITION_3_PRI(val)       vBIT(val,37,3)
#define TX_FIFO_PARTITION_3_LEN(val)       vBIT(val,51,13)

	u64 tx_fifo_partition_2;
#define TX_FIFO_PARTITION_4_PRI(val)       vBIT(val,5,3)
#define TX_FIFO_PARTITION_4_LEN(val)       vBIT(val,19,13)
#define TX_FIFO_PARTITION_5_PRI(val)       vBIT(val,37,3)
#define TX_FIFO_PARTITION_5_LEN(val)       vBIT(val,51,13)

	u64 tx_fifo_partition_3;
#define TX_FIFO_PARTITION_6_PRI(val)       vBIT(val,5,3)
#define TX_FIFO_PARTITION_6_LEN(val)       vBIT(val,19,13)
#define TX_FIFO_PARTITION_7_PRI(val)       vBIT(val,37,3)
#define TX_FIFO_PARTITION_7_LEN(val)       vBIT(val,51,13)

#define TX_FIFO_PARTITION_PRI_0                 0	/* highest */
#define TX_FIFO_PARTITION_PRI_1                 1
#define TX_FIFO_PARTITION_PRI_2                 2
#define TX_FIFO_PARTITION_PRI_3                 3
#define TX_FIFO_PARTITION_PRI_4                 4
#define TX_FIFO_PARTITION_PRI_5                 5
#define TX_FIFO_PARTITION_PRI_6                 6
#define TX_FIFO_PARTITION_PRI_7                 7	/* lowest */

	u64 tx_w_round_robin_0;
	u64 tx_w_round_robin_1;
	u64 tx_w_round_robin_2;
	u64 tx_w_round_robin_3;
	u64 tx_w_round_robin_4;

	u64 tti_command_mem;
#define TTI_CMD_MEM_WE                     s2BIT(7)
#define TTI_CMD_MEM_STROBE_NEW_CMD         s2BIT(15)
#define TTI_CMD_MEM_STROBE_BEING_EXECUTED  s2BIT(15)
#define TTI_CMD_MEM_OFFSET(n)              vBIT(n,26,6)

	u64 tti_data1_mem;
#define TTI_DATA1_MEM_TX_TIMER_VAL(n)      vBIT(n,6,26)
#define TTI_DATA1_MEM_TX_TIMER_AC_CI(n)    vBIT(n,38,2)
#define TTI_DATA1_MEM_TX_TIMER_AC_EN       s2BIT(38)
#define TTI_DATA1_MEM_TX_TIMER_CI_EN       s2BIT(39)
#define TTI_DATA1_MEM_TX_URNG_A(n)         vBIT(n,41,7)
#define TTI_DATA1_MEM_TX_URNG_B(n)         vBIT(n,49,7)
#define TTI_DATA1_MEM_TX_URNG_C(n)         vBIT(n,57,7)

	u64 tti_data2_mem;
#define TTI_DATA2_MEM_TX_UFC_A(n)          vBIT(n,0,16)
#define TTI_DATA2_MEM_TX_UFC_B(n)          vBIT(n,16,16)
#define TTI_DATA2_MEM_TX_UFC_C(n)          vBIT(n,32,16)
#define TTI_DATA2_MEM_TX_UFC_D(n)          vBIT(n,48,16)

/* Tx Protocol assist */
	u64 tx_pa_cfg;
#define TX_PA_CFG_IGNORE_FRM_ERR           s2BIT(1)
#define TX_PA_CFG_IGNORE_SNAP_OUI          s2BIT(2)
#define TX_PA_CFG_IGNORE_LLC_CTRL          s2BIT(3)
#define	TX_PA_CFG_IGNORE_L2_ERR			   s2BIT(6)
#define RX_PA_CFG_STRIP_VLAN_TAG		s2BIT(15)

/* Recent add, used only debug purposes. */
	u64 pcc_enable;

	u8 unused9[0x700 - 0x178];

	u64 txdma_debug_ctrl;

	u8 unused10[0x1800 - 0x1708];

/* RxDMA Registers */
	u64 rxdma_int_status;
	u64 rxdma_int_mask;
#define RXDMA_INT_RC_INT_M             s2BIT(0)
#define RXDMA_INT_RPA_INT_M            s2BIT(1)
#define RXDMA_INT_RDA_INT_M            s2BIT(2)
#define RXDMA_INT_RTI_INT_M            s2BIT(3)

	u64 rda_err_reg;
#define RDA_RXDn_ECC_SG_ERR		vBIT(0xFF,0,8)
#define RDA_RXDn_ECC_DB_ERR		vBIT(0xFF,8,8)
#define RDA_FRM_ECC_SG_ERR		s2BIT(23)
#define RDA_FRM_ECC_DB_N_AERR		s2BIT(31)
#define RDA_SM1_ERR_ALARM		s2BIT(38)
#define RDA_SM0_ERR_ALARM		s2BIT(39)
#define RDA_MISC_ERR			s2BIT(47)
#define RDA_PCIX_ERR			s2BIT(55)
#define RDA_RXD_ECC_DB_SERR		s2BIT(63)
	u64 rda_err_mask;
	u64 rda_err_alarm;

	u64 rc_err_reg;
#define RC_PRCn_ECC_SG_ERR		vBIT(0xFF,0,8)
#define RC_PRCn_ECC_DB_ERR		vBIT(0xFF,8,8)
#define RC_FTC_ECC_SG_ERR		s2BIT(23)
#define RC_FTC_ECC_DB_ERR		s2BIT(31)
#define RC_PRCn_SM_ERR_ALARM		vBIT(0xFF,32,8)
#define RC_FTC_SM_ERR_ALARM		s2BIT(47)
#define RC_RDA_FAIL_WR_Rn		vBIT(0xFF,48,8)
	u64 rc_err_mask;
	u64 rc_err_alarm;

	u64 prc_pcix_err_reg;
#define PRC_PCI_AB_RD_Rn		vBIT(0xFF,0,8)
#define PRC_PCI_DP_RD_Rn		vBIT(0xFF,8,8)
#define PRC_PCI_AB_WR_Rn		vBIT(0xFF,16,8)
#define PRC_PCI_DP_WR_Rn		vBIT(0xFF,24,8)
#define PRC_PCI_AB_F_WR_Rn		vBIT(0xFF,32,8)
#define PRC_PCI_DP_F_WR_Rn		vBIT(0xFF,40,8)
	u64 prc_pcix_err_mask;
	u64 prc_pcix_err_alarm;

	u64 rpa_err_reg;
#define RPA_ECC_SG_ERR			s2BIT(7)
#define RPA_ECC_DB_ERR			s2BIT(15)
#define RPA_FLUSH_REQUEST		s2BIT(22)
#define RPA_SM_ERR_ALARM		s2BIT(23)
#define RPA_CREDIT_ERR			s2BIT(31)
	u64 rpa_err_mask;
	u64 rpa_err_alarm;

	u64 rti_err_reg;
#define RTI_ECC_SG_ERR			s2BIT(7)
#define RTI_ECC_DB_ERR			s2BIT(15)
#define RTI_SM_ERR_ALARM		s2BIT(23)
	u64 rti_err_mask;
	u64 rti_err_alarm;

	u8 unused11[0x100 - 0x88];

/* DMA arbiter */
	u64 rx_queue_priority;
#define RX_QUEUE_0_PRIORITY(val)       vBIT(val,5,3)
#define RX_QUEUE_1_PRIORITY(val)       vBIT(val,13,3)
#define RX_QUEUE_2_PRIORITY(val)       vBIT(val,21,3)
#define RX_QUEUE_3_PRIORITY(val)       vBIT(val,29,3)
#define RX_QUEUE_4_PRIORITY(val)       vBIT(val,37,3)
#define RX_QUEUE_5_PRIORITY(val)       vBIT(val,45,3)
#define RX_QUEUE_6_PRIORITY(val)       vBIT(val,53,3)
#define RX_QUEUE_7_PRIORITY(val)       vBIT(val,61,3)

#define RX_QUEUE_PRI_0                 0	/* highest */
#define RX_QUEUE_PRI_1                 1
#define RX_QUEUE_PRI_2                 2
#define RX_QUEUE_PRI_3                 3
#define RX_QUEUE_PRI_4                 4
#define RX_QUEUE_PRI_5                 5
#define RX_QUEUE_PRI_6                 6
#define RX_QUEUE_PRI_7                 7	/* lowest */

	u64 rx_w_round_robin_0;
	u64 rx_w_round_robin_1;
	u64 rx_w_round_robin_2;
	u64 rx_w_round_robin_3;
	u64 rx_w_round_robin_4;

	/* Per-ring controller regs */
#define RX_MAX_RINGS                8
#if 0
#define RX_MAX_RINGS_SZ             0xFFFF	/* 65536 */
#define RX_MIN_RINGS_SZ             0x3F	/* 63 */
#endif
	u64 prc_rxd0_n[RX_MAX_RINGS];
	u64 prc_ctrl_n[RX_MAX_RINGS];
#define PRC_CTRL_RC_ENABLED                    s2BIT(7)
#define PRC_CTRL_RING_MODE                     (s2BIT(14)|s2BIT(15))
#define PRC_CTRL_RING_MODE_1                   vBIT(0,14,2)
#define PRC_CTRL_RING_MODE_3                   vBIT(1,14,2)
#define PRC_CTRL_RING_MODE_5                   vBIT(2,14,2)
#define PRC_CTRL_RING_MODE_x                   vBIT(3,14,2)
#define PRC_CTRL_NO_SNOOP                      (s2BIT(22)|s2BIT(23))
#define PRC_CTRL_NO_SNOOP_DESC                 s2BIT(22)
#define PRC_CTRL_NO_SNOOP_BUFF                 s2BIT(23)
#define PRC_CTRL_BIMODAL_INTERRUPT             s2BIT(37)
#define PRC_CTRL_GROUP_READS                   s2BIT(38)
#define PRC_CTRL_RXD_BACKOFF_INTERVAL(val)     vBIT(val,40,24)

	u64 prc_alarm_action;
#define PRC_ALARM_ACTION_RR_R0_STOP            s2BIT(3)
#define PRC_ALARM_ACTION_RW_R0_STOP            s2BIT(7)
#define PRC_ALARM_ACTION_RR_R1_STOP            s2BIT(11)
#define PRC_ALARM_ACTION_RW_R1_STOP            s2BIT(15)
#define PRC_ALARM_ACTION_RR_R2_STOP            s2BIT(19)
#define PRC_ALARM_ACTION_RW_R2_STOP            s2BIT(23)
#define PRC_ALARM_ACTION_RR_R3_STOP            s2BIT(27)
#define PRC_ALARM_ACTION_RW_R3_STOP            s2BIT(31)
#define PRC_ALARM_ACTION_RR_R4_STOP            s2BIT(35)
#define PRC_ALARM_ACTION_RW_R4_STOP            s2BIT(39)
#define PRC_ALARM_ACTION_RR_R5_STOP            s2BIT(43)
#define PRC_ALARM_ACTION_RW_R5_STOP            s2BIT(47)
#define PRC_ALARM_ACTION_RR_R6_STOP            s2BIT(51)
#define PRC_ALARM_ACTION_RW_R6_STOP            s2BIT(55)
#define PRC_ALARM_ACTION_RR_R7_STOP            s2BIT(59)
#define PRC_ALARM_ACTION_RW_R7_STOP            s2BIT(63)

/* Receive traffic interrupts */
	u64 rti_command_mem;
#define RTI_CMD_MEM_WE                          s2BIT(7)
#define RTI_CMD_MEM_STROBE                      s2BIT(15)
#define RTI_CMD_MEM_STROBE_NEW_CMD              s2BIT(15)
#define RTI_CMD_MEM_STROBE_CMD_BEING_EXECUTED   s2BIT(15)
#define RTI_CMD_MEM_OFFSET(n)                   vBIT(n,29,3)

	u64 rti_data1_mem;
#define RTI_DATA1_MEM_RX_TIMER_VAL(n)      vBIT(n,3,29)
#define RTI_DATA1_MEM_RX_TIMER_AC_EN       s2BIT(38)
#define RTI_DATA1_MEM_RX_TIMER_CI_EN       s2BIT(39)
#define RTI_DATA1_MEM_RX_URNG_A(n)         vBIT(n,41,7)
#define RTI_DATA1_MEM_RX_URNG_B(n)         vBIT(n,49,7)
#define RTI_DATA1_MEM_RX_URNG_C(n)         vBIT(n,57,7)

	u64 rti_data2_mem;
#define RTI_DATA2_MEM_RX_UFC_A(n)          vBIT(n,0,16)
#define RTI_DATA2_MEM_RX_UFC_B(n)          vBIT(n,16,16)
#define RTI_DATA2_MEM_RX_UFC_C(n)          vBIT(n,32,16)
#define RTI_DATA2_MEM_RX_UFC_D(n)          vBIT(n,48,16)

	u64 rx_pa_cfg;
#define RX_PA_CFG_IGNORE_FRM_ERR           s2BIT(1)
#define RX_PA_CFG_IGNORE_SNAP_OUI          s2BIT(2)
#define RX_PA_CFG_IGNORE_LLC_CTRL          s2BIT(3)
#define RX_PA_CFG_IGNORE_L2_ERR            s2BIT(6)

	u64 unused_11_1;

	u64 ring_bump_counter1;
	u64 ring_bump_counter2;

	u8 unused12[0x700 - 0x1F0];

	u64 rxdma_debug_ctrl;

	u8 unused13[0x2000 - 0x1f08];

/* Media Access Controller Register */
	u64 mac_int_status;
	u64 mac_int_mask;
#define MAC_INT_STATUS_TMAC_INT            s2BIT(0)
#define MAC_INT_STATUS_RMAC_INT            s2BIT(1)

	u64 mac_tmac_err_reg;
#define TMAC_ECC_SG_ERR				s2BIT(7)
#define TMAC_ECC_DB_ERR				s2BIT(15)
#define TMAC_TX_BUF_OVRN			s2BIT(23)
#define TMAC_TX_CRI_ERR				s2BIT(31)
#define TMAC_TX_SM_ERR				s2BIT(39)
#define TMAC_DESC_ECC_SG_ERR			s2BIT(47)
#define TMAC_DESC_ECC_DB_ERR			s2BIT(55)

	u64 mac_tmac_err_mask;
	u64 mac_tmac_err_alarm;

	u64 mac_rmac_err_reg;
#define RMAC_RX_BUFF_OVRN			s2BIT(0)
#define RMAC_FRM_RCVD_INT			s2BIT(1)
#define RMAC_UNUSED_INT				s2BIT(2)
#define RMAC_RTS_PNUM_ECC_SG_ERR		s2BIT(5)
#define RMAC_RTS_DS_ECC_SG_ERR			s2BIT(6)
#define RMAC_RD_BUF_ECC_SG_ERR			s2BIT(7)
#define RMAC_RTH_MAP_ECC_SG_ERR			s2BIT(8)
#define RMAC_RTH_SPDM_ECC_SG_ERR		s2BIT(9)
#define RMAC_RTS_VID_ECC_SG_ERR			s2BIT(10)
#define RMAC_DA_SHADOW_ECC_SG_ERR		s2BIT(11)
#define RMAC_RTS_PNUM_ECC_DB_ERR		s2BIT(13)
#define RMAC_RTS_DS_ECC_DB_ERR			s2BIT(14)
#define RMAC_RD_BUF_ECC_DB_ERR			s2BIT(15)
#define RMAC_RTH_MAP_ECC_DB_ERR			s2BIT(16)
#define RMAC_RTH_SPDM_ECC_DB_ERR		s2BIT(17)
#define RMAC_RTS_VID_ECC_DB_ERR			s2BIT(18)
#define RMAC_DA_SHADOW_ECC_DB_ERR		s2BIT(19)
#define RMAC_LINK_STATE_CHANGE_INT		s2BIT(31)
#define RMAC_RX_SM_ERR				s2BIT(39)
#define RMAC_SINGLE_ECC_ERR			(s2BIT(5) | s2BIT(6) | s2BIT(7) |\
						s2BIT(8)  | s2BIT(9) | s2BIT(10)|\
						s2BIT(11))
#define RMAC_DOUBLE_ECC_ERR			(s2BIT(13) | s2BIT(14) | s2BIT(15) |\
						s2BIT(16)  | s2BIT(17) | s2BIT(18)|\
						s2BIT(19))
	u64 mac_rmac_err_mask;
	u64 mac_rmac_err_alarm;

	u8 unused14[0x100 - 0x40];

	u64 mac_cfg;
#define MAC_CFG_TMAC_ENABLE             s2BIT(0)
#define MAC_CFG_RMAC_ENABLE             s2BIT(1)
#define MAC_CFG_LAN_NOT_WAN             s2BIT(2)
#define MAC_CFG_TMAC_LOOPBACK           s2BIT(3)
#define MAC_CFG_TMAC_APPEND_PAD         s2BIT(4)
#define MAC_CFG_RMAC_STRIP_FCS          s2BIT(5)
#define MAC_CFG_RMAC_STRIP_PAD          s2BIT(6)
#define MAC_CFG_RMAC_PROM_ENABLE        s2BIT(7)
#define MAC_RMAC_DISCARD_PFRM           s2BIT(8)
#define MAC_RMAC_BCAST_ENABLE           s2BIT(9)
#define MAC_RMAC_ALL_ADDR_ENABLE        s2BIT(10)
#define MAC_RMAC_INVLD_IPG_THR(val)     vBIT(val,16,8)

	u64 tmac_avg_ipg;
#define TMAC_AVG_IPG(val)           vBIT(val,0,8)

	u64 rmac_max_pyld_len;
#define RMAC_MAX_PYLD_LEN(val)      vBIT(val,2,14)
#define RMAC_MAX_PYLD_LEN_DEF       vBIT(1500,2,14)
#define RMAC_MAX_PYLD_LEN_JUMBO_DEF vBIT(9600,2,14)

	u64 rmac_err_cfg;
#define RMAC_ERR_FCS                    s2BIT(0)
#define RMAC_ERR_FCS_ACCEPT             s2BIT(1)
#define RMAC_ERR_TOO_LONG               s2BIT(1)
#define RMAC_ERR_TOO_LONG_ACCEPT        s2BIT(1)
#define RMAC_ERR_RUNT                   s2BIT(2)
#define RMAC_ERR_RUNT_ACCEPT            s2BIT(2)
#define RMAC_ERR_LEN_MISMATCH           s2BIT(3)
#define RMAC_ERR_LEN_MISMATCH_ACCEPT    s2BIT(3)

	u64 rmac_cfg_key;
#define RMAC_CFG_KEY(val)               vBIT(val,0,16)

#define S2IO_MAC_ADDR_START_OFFSET	0

#define S2IO_XENA_MAX_MC_ADDRESSES	64	/* multicast addresses */
#define S2IO_HERC_MAX_MC_ADDRESSES	256

#define S2IO_XENA_MAX_MAC_ADDRESSES	16
#define S2IO_HERC_MAX_MAC_ADDRESSES	64

#define S2IO_XENA_MC_ADDR_START_OFFSET	16
#define S2IO_HERC_MC_ADDR_START_OFFSET	64

	u64 rmac_addr_cmd_mem;
#define RMAC_ADDR_CMD_MEM_WE                    s2BIT(7)
#define RMAC_ADDR_CMD_MEM_RD                    0
#define RMAC_ADDR_CMD_MEM_STROBE_NEW_CMD        s2BIT(15)
#define RMAC_ADDR_CMD_MEM_STROBE_CMD_EXECUTING  s2BIT(15)
#define RMAC_ADDR_CMD_MEM_OFFSET(n)             vBIT(n,26,6)

	u64 rmac_addr_data0_mem;
#define RMAC_ADDR_DATA0_MEM_ADDR(n)    vBIT(n,0,48)
#define RMAC_ADDR_DATA0_MEM_USER       s2BIT(48)

	u64 rmac_addr_data1_mem;
#define RMAC_ADDR_DATA1_MEM_MASK(n)    vBIT(n,0,48)

	u8 unused15[0x8];

/*
        u64 rmac_addr_cfg;
#define RMAC_ADDR_UCASTn_EN(n)     mBIT(0)_n(n)
#define RMAC_ADDR_MCASTn_EN(n)     mBIT(0)_n(n)
#define RMAC_ADDR_BCAST_EN         vBIT(0)_48
#define RMAC_ADDR_ALL_ADDR_EN      vBIT(0)_49
*/
	u64 tmac_ipg_cfg;

	u64 rmac_pause_cfg;
#define RMAC_PAUSE_GEN             s2BIT(0)
#define RMAC_PAUSE_GEN_ENABLE      s2BIT(0)
#define RMAC_PAUSE_RX              s2BIT(1)
#define RMAC_PAUSE_RX_ENABLE       s2BIT(1)
#define RMAC_PAUSE_HG_PTIME_DEF    vBIT(0xFFFF,16,16)
#define RMAC_PAUSE_HG_PTIME(val)    vBIT(val,16,16)

	u64 rmac_red_cfg;

	u64 rmac_red_rate_q0q3;
	u64 rmac_red_rate_q4q7;

	u64 mac_link_util;
#define MAC_TX_LINK_UTIL           vBIT(0xFE,1,7)
#define MAC_TX_LINK_UTIL_DISABLE   vBIT(0xF, 8,4)
#define MAC_TX_LINK_UTIL_VAL( n )  vBIT(n,8,4)
#define MAC_RX_LINK_UTIL           vBIT(0xFE,33,7)
#define MAC_RX_LINK_UTIL_DISABLE   vBIT(0xF,40,4)
#define MAC_RX_LINK_UTIL_VAL( n )  vBIT(n,40,4)

#define MAC_LINK_UTIL_DISABLE      MAC_TX_LINK_UTIL_DISABLE | \
                                   MAC_RX_LINK_UTIL_DISABLE

	u64 rmac_invalid_ipg;

/* rx traffic steering */
#define	MAC_RTS_FRM_LEN_SET(len)	vBIT(len,2,14)
	u64 rts_frm_len_n[8];

	u64 rts_qos_steering;

#define MAX_DIX_MAP                         4
	u64 rts_dix_map_n[MAX_DIX_MAP];
#define RTS_DIX_MAP_ETYPE(val)             vBIT(val,0,16)
#define RTS_DIX_MAP_SCW(val)               s2BIT(val,21)

	u64 rts_q_alternates;
	u64 rts_default_q;

	u64 rts_ctrl;
#define RTS_CTRL_IGNORE_SNAP_OUI           s2BIT(2)
#define RTS_CTRL_IGNORE_LLC_CTRL           s2BIT(3)

	u64 rts_pn_cam_ctrl;
#define RTS_PN_CAM_CTRL_WE                 s2BIT(7)
#define RTS_PN_CAM_CTRL_STROBE_NEW_CMD     s2BIT(15)
#define RTS_PN_CAM_CTRL_STROBE_BEING_EXECUTED   s2BIT(15)
#define RTS_PN_CAM_CTRL_OFFSET(n)          vBIT(n,24,8)
	u64 rts_pn_cam_data;
#define RTS_PN_CAM_DATA_TCP_SELECT         s2BIT(7)
#define RTS_PN_CAM_DATA_PORT(val)          vBIT(val,8,16)
#define RTS_PN_CAM_DATA_SCW(val)           vBIT(val,24,8)

	u64 rts_ds_mem_ctrl;
#define RTS_DS_MEM_CTRL_WE                 s2BIT(7)
#define RTS_DS_MEM_CTRL_STROBE_NEW_CMD     s2BIT(15)
#define RTS_DS_MEM_CTRL_STROBE_CMD_BEING_EXECUTED   s2BIT(15)
#define RTS_DS_MEM_CTRL_OFFSET(n)          vBIT(n,26,6)
	u64 rts_ds_mem_data;
#define RTS_DS_MEM_DATA(n)                 vBIT(n,0,8)

	u8 unused16[0x700 - 0x220];

	u64 mac_debug_ctrl;
#define MAC_DBG_ACTIVITY_VALUE		   0x411040400000000ULL

	u8 unused17[0x2800 - 0x2708];

/* memory controller registers */
	u64 mc_int_status;
#define MC_INT_STATUS_MC_INT               s2BIT(0)
	u64 mc_int_mask;
#define MC_INT_MASK_MC_INT                 s2BIT(0)

	u64 mc_err_reg;
#define MC_ERR_REG_ECC_DB_ERR_L            s2BIT(14)
#define MC_ERR_REG_ECC_DB_ERR_U            s2BIT(15)
#define MC_ERR_REG_MIRI_ECC_DB_ERR_0       s2BIT(18)
#define MC_ERR_REG_MIRI_ECC_DB_ERR_1       s2BIT(20)
#define MC_ERR_REG_MIRI_CRI_ERR_0          s2BIT(22)
#define MC_ERR_REG_MIRI_CRI_ERR_1          s2BIT(23)
#define MC_ERR_REG_SM_ERR                  s2BIT(31)
#define MC_ERR_REG_ECC_ALL_SNG		   (s2BIT(2) | s2BIT(3) | s2BIT(4) | s2BIT(5) |\
					s2BIT(17) | s2BIT(19))
#define MC_ERR_REG_ECC_ALL_DBL		   (s2BIT(10) | s2BIT(11) | s2BIT(12) |\
					s2BIT(13) | s2BIT(18) | s2BIT(20))
#define PLL_LOCK_N			s2BIT(39)
	u64 mc_err_mask;
	u64 mc_err_alarm;

	u8 unused18[0x100 - 0x28];

/* MC configuration */
	u64 rx_queue_cfg;
#define RX_QUEUE_CFG_Q0_SZ(n)              vBIT(n,0,8)
#define RX_QUEUE_CFG_Q1_SZ(n)              vBIT(n,8,8)
#define RX_QUEUE_CFG_Q2_SZ(n)              vBIT(n,16,8)
#define RX_QUEUE_CFG_Q3_SZ(n)              vBIT(n,24,8)
#define RX_QUEUE_CFG_Q4_SZ(n)              vBIT(n,32,8)
#define RX_QUEUE_CFG_Q5_SZ(n)              vBIT(n,40,8)
#define RX_QUEUE_CFG_Q6_SZ(n)              vBIT(n,48,8)
#define RX_QUEUE_CFG_Q7_SZ(n)              vBIT(n,56,8)

	u64 mc_rldram_mrs;
#define	MC_RLDRAM_QUEUE_SIZE_ENABLE			s2BIT(39)
#define	MC_RLDRAM_MRS_ENABLE				s2BIT(47)

	u64 mc_rldram_interleave;

	u64 mc_pause_thresh_q0q3;
	u64 mc_pause_thresh_q4q7;

	u64 mc_red_thresh_q[8];

	u8 unused19[0x200 - 0x168];
	u64 mc_rldram_ref_per;
	u8 unused20[0x220 - 0x208];
	u64 mc_rldram_test_ctrl;
#define MC_RLDRAM_TEST_MODE		s2BIT(47)
#define MC_RLDRAM_TEST_WRITE	s2BIT(7)
#define MC_RLDRAM_TEST_GO		s2BIT(15)
#define MC_RLDRAM_TEST_DONE		s2BIT(23)
#define MC_RLDRAM_TEST_PASS		s2BIT(31)

	u8 unused21[0x240 - 0x228];
	u64 mc_rldram_test_add;
	u8 unused22[0x260 - 0x248];
	u64 mc_rldram_test_d0;
	u8 unused23[0x280 - 0x268];
	u64 mc_rldram_test_d1;
	u8 unused24[0x300 - 0x288];
	u64 mc_rldram_test_d2;

	u8 unused24_1[0x360 - 0x308];
	u64 mc_rldram_ctrl;
#define	MC_RLDRAM_ENABLE_ODT		s2BIT(7)

	u8 unused24_2[0x640 - 0x368];
	u64 mc_rldram_ref_per_herc;
#define	MC_RLDRAM_SET_REF_PERIOD(val)	vBIT(val, 0, 16)

	u8 unused24_3[0x660 - 0x648];
	u64 mc_rldram_mrs_herc;

	u8 unused25[0x700 - 0x668];
	u64 mc_debug_ctrl;

	u8 unused26[0x3000 - 0x2f08];

/* XGXG */
	/* XGXS control registers */

	u64 xgxs_int_status;
#define XGXS_INT_STATUS_TXGXS              s2BIT(0)
#define XGXS_INT_STATUS_RXGXS              s2BIT(1)
	u64 xgxs_int_mask;
#define XGXS_INT_MASK_TXGXS                s2BIT(0)
#define XGXS_INT_MASK_RXGXS                s2BIT(1)

	u64 xgxs_txgxs_err_reg;
#define TXGXS_ECC_SG_ERR		s2BIT(7)
#define TXGXS_ECC_DB_ERR		s2BIT(15)
#define TXGXS_ESTORE_UFLOW		s2BIT(31)
#define TXGXS_TX_SM_ERR			s2BIT(39)

	u64 xgxs_txgxs_err_mask;
	u64 xgxs_txgxs_err_alarm;

	u64 xgxs_rxgxs_err_reg;
#define RXGXS_ESTORE_OFLOW		s2BIT(7)
#define RXGXS_RX_SM_ERR			s2BIT(39)
	u64 xgxs_rxgxs_err_mask;
	u64 xgxs_rxgxs_err_alarm;

	u8 unused27[0x100 - 0x40];

	u64 xgxs_cfg;
	u64 xgxs_status;

	u64 xgxs_cfg_key;
	u64 xgxs_efifo_cfg;	/* CHANGED */
	u64 rxgxs_ber_0;	/* CHANGED */
	u64 rxgxs_ber_1;	/* CHANGED */

	u64 spi_control;
#define SPI_CONTROL_KEY(key)		vBIT(key,0,4)
#define SPI_CONTROL_BYTECNT(cnt)	vBIT(cnt,29,3)
#define SPI_CONTROL_CMD(cmd)		vBIT(cmd,32,8)
#define SPI_CONTROL_ADDR(addr)		vBIT(addr,40,24)
#define SPI_CONTROL_SEL1		s2BIT(4)
#define SPI_CONTROL_REQ			s2BIT(7)
#define SPI_CONTROL_NACK		s2BIT(5)
#define SPI_CONTROL_DONE		s2BIT(6)
	u64 spi_data;
#define SPI_DATA_WRITE(data,len)	vBIT(data,0,len)
};

#define XENA_REG_SPACE	sizeof(struct XENA_dev_config)
#define	XENA_EEPROM_SPACE (0x01 << 11)

#endif				/* _REGS_H */
>2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041
/*
 *	Intel IO-APIC support for multi-Pentium hosts.
 *
 *	Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
 *
 *	Many thanks to Stig Venaas for trying out countless experimental
 *	patches and reporting/debugging problems patiently!
 *
 *	(c) 1999, Multiple IO-APIC support, developed by
 *	Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
 *      Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
 *	further tested and cleaned up by Zach Brown <zab@redhat.com>
 *	and Ingo Molnar <mingo@redhat.com>
 *
 *	Fixes
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs;
 *					thanks to Eric Gilmore
 *					and Rolf G. Tews
 *					for testing these extensively
 *	Paul Diefenbaugh	:	Added full ACPI support
 */

#include <linux/mm.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
#include <linux/pci.h>
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
#include <linux/acpi.h>
#include <linux/module.h>
#include <linux/sysdev.h>
#include <linux/msi.h>
#include <linux/htirq.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/jiffies.h>	/* time_after() */
#ifdef CONFIG_ACPI
#include <acpi/acpi_bus.h>
#endif
#include <linux/bootmem.h>
#include <linux/dmar.h>
#include <linux/hpet.h>

#include <asm/idle.h>
#include <asm/io.h>
#include <asm/smp.h>
#include <asm/desc.h>
#include <asm/proto.h>
#include <asm/acpi.h>
#include <asm/dma.h>
#include <asm/timer.h>
#include <asm/i8259.h>
#include <asm/nmi.h>
#include <asm/msidef.h>
#include <asm/hypertransport.h>
#include <asm/setup.h>
#include <asm/irq_remapping.h>
#include <asm/hpet.h>
#include <asm/uv/uv_hub.h>
#include <asm/uv/uv_irq.h>

#include <mach_ipi.h>
#include <mach_apic.h>
#include <mach_apicdef.h>

#define __apicdebuginit(type) static type __init

/*
 *      Is the SiS APIC rmw bug present ?
 *      -1 = don't know, 0 = no, 1 = yes
 */
int sis_apic_bug = -1;

static DEFINE_SPINLOCK(ioapic_lock);
static DEFINE_SPINLOCK(vector_lock);

/*
 * # of IRQ routing registers
 */
int nr_ioapic_registers[MAX_IO_APICS];

/* I/O APIC entries */
struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
int nr_ioapics;

/* MP IRQ source entries */
struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];

/* # of MP IRQ source entries */
int mp_irq_entries;

#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
int mp_bus_id_to_type[MAX_MP_BUSSES];
#endif

DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);

int skip_ioapic_setup;

static int __init parse_noapic(char *str)
{
	/* disable IO-APIC */
	disable_ioapic_setup();
	return 0;
}
early_param("noapic", parse_noapic);

struct irq_pin_list;

/*
 * This is performance-critical, we want to do it O(1)
 *
 * the indexing order of this array favors 1:1 mappings
 * between pins and IRQs.
 */

struct irq_pin_list {
	int apic, pin;
	struct irq_pin_list *next;
};

static struct irq_pin_list *get_one_free_irq_2_pin(int cpu)
{
	struct irq_pin_list *pin;
	int node;

	node = cpu_to_node(cpu);

	pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
	printk(KERN_DEBUG "  alloc irq_2_pin on cpu %d node %d\n", cpu, node);

	return pin;
}

struct irq_cfg {
	struct irq_pin_list *irq_2_pin;
	cpumask_t domain;
	cpumask_t old_domain;
	unsigned move_cleanup_count;
	u8 vector;
	u8 move_in_progress : 1;
};

/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
#ifdef CONFIG_SPARSE_IRQ
static struct irq_cfg irq_cfgx[] = {
#else
static struct irq_cfg irq_cfgx[NR_IRQS] = {
#endif
	[0]  = { .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR,  },
	[1]  = { .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR,  },
	[2]  = { .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR,  },
	[3]  = { .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR,  },
	[4]  = { .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR,  },
	[5]  = { .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR,  },
	[6]  = { .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR,  },
	[7]  = { .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR,  },
	[8]  = { .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR,  },
	[9]  = { .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR,  },
	[10] = { .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
	[11] = { .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
	[12] = { .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
	[13] = { .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
	[14] = { .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
	[15] = { .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
};

void __init arch_early_irq_init(void)
{
	struct irq_cfg *cfg;
	struct irq_desc *desc;
	int count;
	int i;

	cfg = irq_cfgx;
	count = ARRAY_SIZE(irq_cfgx);

	for (i = 0; i < count; i++) {
		desc = irq_to_desc(i);
		desc->chip_data = &cfg[i];
	}
}

#ifdef CONFIG_SPARSE_IRQ
static struct irq_cfg *irq_cfg(unsigned int irq)
{
	struct irq_cfg *cfg = NULL;
	struct irq_desc *desc;

	desc = irq_to_desc(irq);
	if (desc)
		cfg = desc->chip_data;

	return cfg;
}

static struct irq_cfg *get_one_free_irq_cfg(int cpu)
{
	struct irq_cfg *cfg;
	int node;

	node = cpu_to_node(cpu);

	cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
	printk(KERN_DEBUG "  alloc irq_cfg on cpu %d node %d\n", cpu, node);

	return cfg;
}

void arch_init_chip_data(struct irq_desc *desc, int cpu)
{
	struct irq_cfg *cfg;

	cfg = desc->chip_data;
	if (!cfg) {
		desc->chip_data = get_one_free_irq_cfg(cpu);
		if (!desc->chip_data) {
			printk(KERN_ERR "can not alloc irq_cfg\n");
			BUG_ON(1);
		}
	}
}

#else
static struct irq_cfg *irq_cfg(unsigned int irq)
{
	return irq < nr_irqs ? irq_cfgx + irq : NULL;
}

#endif

static inline void set_extra_move_desc(struct irq_desc *desc, cpumask_t mask)
{
}

struct io_apic {
	unsigned int index;
	unsigned int unused[3];
	unsigned int data;
};

static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
{
	return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
		+ (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
}

static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	return readl(&io_apic->data);
}

static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);
	writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

/*
 * Re-write a value: to be used for read-modify-write
 * cycles where the read already set up the index register.
 *
 * Older SiS APIC requires we rewrite the index register
 */
static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
{
	struct io_apic __iomem *io_apic = io_apic_base(apic);

	if (sis_apic_bug)
		writel(reg, &io_apic->index);
	writel(value, &io_apic->data);
}

static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
{
	struct irq_pin_list *entry;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	entry = cfg->irq_2_pin;
	for (;;) {
		unsigned int reg;
		int pin;

		if (!entry)
			break;
		pin = entry->pin;
		reg = io_apic_read(entry->apic, 0x10 + pin*2);
		/* Is the remote IRR bit set? */
		if (reg & IO_APIC_REDIR_REMOTE_IRR) {
			spin_unlock_irqrestore(&ioapic_lock, flags);
			return true;
		}
		if (!entry->next)
			break;
		entry = entry->next;
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return false;
}

union entry_union {
	struct { u32 w1, w2; };
	struct IO_APIC_route_entry entry;
};

static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
{
	union entry_union eu;
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
	eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
	spin_unlock_irqrestore(&ioapic_lock, flags);
	return eu.entry;
}

/*
 * When we write a new IO APIC routing entry, we need to write the high
 * word first! If the mask bit in the low word is clear, we will enable
 * the interrupt, and we need to make sure the entry is fully populated
 * before that happens.
 */
static void
__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
{
	union entry_union eu;
	eu.entry = e;
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
}

static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
{
	unsigned long flags;
	spin_lock_irqsave(&ioapic_lock, flags);
	__ioapic_write_entry(apic, pin, e);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

/*
 * When we mask an IO APIC routing entry, we need to write the low
 * word first, in order to set the mask bit before we change the
 * high bits!
 */
static void ioapic_mask_entry(int apic, int pin)
{
	unsigned long flags;
	union entry_union eu = { .entry.mask = 1 };

	spin_lock_irqsave(&ioapic_lock, flags);
	io_apic_write(apic, 0x10 + 2*pin, eu.w1);
	io_apic_write(apic, 0x11 + 2*pin, eu.w2);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

#ifdef CONFIG_SMP
static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
{
	int apic, pin;
	struct irq_pin_list *entry;
	u8 vector = cfg->vector;

	entry = cfg->irq_2_pin;
	for (;;) {
		unsigned int reg;

		if (!entry)
			break;

		apic = entry->apic;
		pin = entry->pin;
#ifdef CONFIG_INTR_REMAP
		/*
		 * With interrupt-remapping, destination information comes
		 * from interrupt-remapping table entry.
		 */
		if (!irq_remapped(irq))
			io_apic_write(apic, 0x11 + pin*2, dest);
#else
		io_apic_write(apic, 0x11 + pin*2, dest);
#endif
		reg = io_apic_read(apic, 0x10 + pin*2);
		reg &= ~IO_APIC_REDIR_VECTOR_MASK;
		reg |= vector;
		io_apic_modify(apic, 0x10 + pin*2, reg);
		if (!entry->next)
			break;
		entry = entry->next;
	}
}

static int assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask);

static void set_ioapic_affinity_irq_desc(struct irq_desc *desc, cpumask_t mask)
{
	struct irq_cfg *cfg;
	unsigned long flags;
	unsigned int dest;
	cpumask_t tmp;
	unsigned int irq;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	irq = desc->irq;
	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	set_extra_move_desc(desc, mask);

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);
	/*
	 * Only the high 8 bits are valid.
	 */
	dest = SET_APIC_LOGICAL_ID(dest);

	spin_lock_irqsave(&ioapic_lock, flags);
	__target_IO_APIC_irq(irq, dest, cfg);
	desc->affinity = mask;
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc;

	desc = irq_to_desc(irq);

	set_ioapic_affinity_irq_desc(desc, mask);
}
#endif /* CONFIG_SMP */

/*
 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
 * shared ISA-space IRQs, so we have to support them. We are super
 * fast in the common case, and fast for shared ISA-space IRQs.
 */
static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
{
	struct irq_pin_list *entry;

	entry = cfg->irq_2_pin;
	if (!entry) {
		entry = get_one_free_irq_2_pin(cpu);
		if (!entry) {
			printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
					apic, pin);
			return;
		}
		cfg->irq_2_pin = entry;
		entry->apic = apic;
		entry->pin = pin;
		return;
	}

	while (entry->next) {
		/* not again, please */
		if (entry->apic == apic && entry->pin == pin)
			return;

		entry = entry->next;
	}

	entry->next = get_one_free_irq_2_pin(cpu);
	entry = entry->next;
	entry->apic = apic;
	entry->pin = pin;
}

/*
 * Reroute an IRQ to a different pin.
 */
static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu,
				      int oldapic, int oldpin,
				      int newapic, int newpin)
{
	struct irq_pin_list *entry = cfg->irq_2_pin;
	int replaced = 0;

	while (entry) {
		if (entry->apic == oldapic && entry->pin == oldpin) {
			entry->apic = newapic;
			entry->pin = newpin;
			replaced = 1;
			/* every one is different, right? */
			break;
		}
		entry = entry->next;
	}

	/* why? call replace before add? */
	if (!replaced)
		add_pin_to_irq_cpu(cfg, cpu, newapic, newpin);
}

static inline void io_apic_modify_irq(struct irq_cfg *cfg,
				int mask_and, int mask_or,
				void (*final)(struct irq_pin_list *entry))
{
	int pin;
	struct irq_pin_list *entry;

	for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) {
		unsigned int reg;
		pin = entry->pin;
		reg = io_apic_read(entry->apic, 0x10 + pin * 2);
		reg &= mask_and;
		reg |= mask_or;
		io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
		if (final)
			final(entry);
	}
}

static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
{
	io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
}

#ifdef CONFIG_X86_64
void io_apic_sync(struct irq_pin_list *entry)
{
	/*
	 * Synchronize the IO-APIC and the CPU by doing
	 * a dummy read from the IO-APIC
	 */
	struct io_apic __iomem *io_apic;
	io_apic = io_apic_base(entry->apic);
	readl(&io_apic->data);
}

static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
{
	io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
}
#else /* CONFIG_X86_32 */
static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
{
	io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, NULL);
}

static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
{
	io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER,
			IO_APIC_REDIR_MASKED, NULL);
}

static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
{
	io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
			IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
}
#endif /* CONFIG_X86_32 */

static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
{
	struct irq_cfg *cfg = desc->chip_data;
	unsigned long flags;

	BUG_ON(!cfg);

	spin_lock_irqsave(&ioapic_lock, flags);
	__mask_IO_APIC_irq(cfg);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
{
	struct irq_cfg *cfg = desc->chip_data;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	__unmask_IO_APIC_irq(cfg);
	spin_unlock_irqrestore(&ioapic_lock, flags);
}

static void mask_IO_APIC_irq(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	mask_IO_APIC_irq_desc(desc);
}
static void unmask_IO_APIC_irq(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	unmask_IO_APIC_irq_desc(desc);
}

static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
{
	struct IO_APIC_route_entry entry;

	/* Check delivery_mode to be sure we're not clearing an SMI pin */
	entry = ioapic_read_entry(apic, pin);
	if (entry.delivery_mode == dest_SMI)
		return;
	/*
	 * Disable it in the IO-APIC irq-routing table:
	 */
	ioapic_mask_entry(apic, pin);
}

static void clear_IO_APIC (void)
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			clear_IO_APIC_pin(apic, pin);
}

#if !defined(CONFIG_SMP) && defined(CONFIG_X86_32)
void send_IPI_self(int vector)
{
	unsigned int cfg;

	/*
	 * Wait for idle.
	 */
	apic_wait_icr_idle();
	cfg = APIC_DM_FIXED | APIC_DEST_SELF | vector | APIC_DEST_LOGICAL;
	/*
	 * Send the IPI. The write to APIC_ICR fires this off.
	 */
	apic_write(APIC_ICR, cfg);
}
#endif /* !CONFIG_SMP && CONFIG_X86_32*/

#ifdef CONFIG_X86_32
/*
 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
 * specific CPU-side IRQs.
 */

#define MAX_PIRQS 8
static int pirq_entries [MAX_PIRQS];
static int pirqs_enabled;

static int __init ioapic_pirq_setup(char *str)
{
	int i, max;
	int ints[MAX_PIRQS+1];

	get_options(str, ARRAY_SIZE(ints), ints);

	for (i = 0; i < MAX_PIRQS; i++)
		pirq_entries[i] = -1;

	pirqs_enabled = 1;
	apic_printk(APIC_VERBOSE, KERN_INFO
			"PIRQ redirection, working around broken MP-BIOS.\n");
	max = MAX_PIRQS;
	if (ints[0] < MAX_PIRQS)
		max = ints[0];

	for (i = 0; i < max; i++) {
		apic_printk(APIC_VERBOSE, KERN_DEBUG
				"... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
		/*
		 * PIRQs are mapped upside down, usually.
		 */
		pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
	}
	return 1;
}

__setup("pirq=", ioapic_pirq_setup);
#endif /* CONFIG_X86_32 */

#ifdef CONFIG_INTR_REMAP
/* I/O APIC RTE contents at the OS boot up */
static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];

/*
 * Saves and masks all the unmasked IO-APIC RTE's
 */
int save_mask_IO_APIC_setup(void)
{
	union IO_APIC_reg_01 reg_01;
	unsigned long flags;
	int apic, pin;

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
	for (apic = 0; apic < nr_ioapics; apic++) {
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_01.raw = io_apic_read(apic, 1);
		spin_unlock_irqrestore(&ioapic_lock, flags);
		nr_ioapic_registers[apic] = reg_01.bits.entries+1;
	}

	for (apic = 0; apic < nr_ioapics; apic++) {
		early_ioapic_entries[apic] =
			kzalloc(sizeof(struct IO_APIC_route_entry) *
				nr_ioapic_registers[apic], GFP_KERNEL);
		if (!early_ioapic_entries[apic])
			goto nomem;
	}

	for (apic = 0; apic < nr_ioapics; apic++)
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			struct IO_APIC_route_entry entry;

			entry = early_ioapic_entries[apic][pin] =
				ioapic_read_entry(apic, pin);
			if (!entry.mask) {
				entry.mask = 1;
				ioapic_write_entry(apic, pin, entry);
			}
		}

	return 0;

nomem:
	while (apic >= 0)
		kfree(early_ioapic_entries[apic--]);
	memset(early_ioapic_entries, 0,
		ARRAY_SIZE(early_ioapic_entries));

	return -ENOMEM;
}

void restore_IO_APIC_setup(void)
{
	int apic, pin;

	for (apic = 0; apic < nr_ioapics; apic++) {
		if (!early_ioapic_entries[apic])
			break;
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
			ioapic_write_entry(apic, pin,
					   early_ioapic_entries[apic][pin]);
		kfree(early_ioapic_entries[apic]);
		early_ioapic_entries[apic] = NULL;
	}
}

void reinit_intr_remapped_IO_APIC(int intr_remapping)
{
	/*
	 * for now plain restore of previous settings.
	 * TBD: In the case of OS enabling interrupt-remapping,
	 * IO-APIC RTE's need to be setup to point to interrupt-remapping
	 * table entries. for now, do a plain restore, and wait for
	 * the setup_IO_APIC_irqs() to do proper initialization.
	 */
	restore_IO_APIC_setup();
}
#endif

/*
 * Find the IRQ entry number of a certain pin.
 */
static int find_irq_entry(int apic, int pin, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++)
		if (mp_irqs[i].mp_irqtype == type &&
		    (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
		     mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
		    mp_irqs[i].mp_dstirq == pin)
			return i;

	return -1;
}

/*
 * Find the pin to which IRQ[irq] (ISA) is connected
 */
static int __init find_isa_irq_pin(int irq, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
		int lbus = mp_irqs[i].mp_srcbus;

		if (test_bit(lbus, mp_bus_not_pci) &&
		    (mp_irqs[i].mp_irqtype == type) &&
		    (mp_irqs[i].mp_srcbusirq == irq))

			return mp_irqs[i].mp_dstirq;
	}
	return -1;
}

static int __init find_isa_irq_apic(int irq, int type)
{
	int i;

	for (i = 0; i < mp_irq_entries; i++) {
		int lbus = mp_irqs[i].mp_srcbus;

		if (test_bit(lbus, mp_bus_not_pci) &&
		    (mp_irqs[i].mp_irqtype == type) &&
		    (mp_irqs[i].mp_srcbusirq == irq))
			break;
	}
	if (i < mp_irq_entries) {
		int apic;
		for(apic = 0; apic < nr_ioapics; apic++) {
			if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
				return apic;
		}
	}

	return -1;
}

/*
 * Find a specific PCI IRQ entry.
 * Not an __init, possibly needed by modules
 */
static int pin_2_irq(int idx, int apic, int pin);

int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
{
	int apic, i, best_guess = -1;

	apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
		bus, slot, pin);
	if (test_bit(bus, mp_bus_not_pci)) {
		apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
		return -1;
	}
	for (i = 0; i < mp_irq_entries; i++) {
		int lbus = mp_irqs[i].mp_srcbus;

		for (apic = 0; apic < nr_ioapics; apic++)
			if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
			    mp_irqs[i].mp_dstapic == MP_APIC_ALL)
				break;

		if (!test_bit(lbus, mp_bus_not_pci) &&
		    !mp_irqs[i].mp_irqtype &&
		    (bus == lbus) &&
		    (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
			int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);

			if (!(apic || IO_APIC_IRQ(irq)))
				continue;

			if (pin == (mp_irqs[i].mp_srcbusirq & 3))
				return irq;
			/*
			 * Use the first all-but-pin matching entry as a
			 * best-guess fuzzy result for broken mptables.
			 */
			if (best_guess < 0)
				best_guess = irq;
		}
	}
	return best_guess;
}

EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);

#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
/*
 * EISA Edge/Level control register, ELCR
 */
static int EISA_ELCR(unsigned int irq)
{
	if (irq < NR_IRQS_LEGACY) {
		unsigned int port = 0x4d0 + (irq >> 3);
		return (inb(port) >> (irq & 7)) & 1;
	}
	apic_printk(APIC_VERBOSE, KERN_INFO
			"Broken MPtable reports ISA irq %d\n", irq);
	return 0;
}

#endif

/* ISA interrupts are always polarity zero edge triggered,
 * when listed as conforming in the MP table. */

#define default_ISA_trigger(idx)	(0)
#define default_ISA_polarity(idx)	(0)

/* EISA interrupts are always polarity zero and can be edge or level
 * trigger depending on the ELCR value.  If an interrupt is listed as
 * EISA conforming in the MP table, that means its trigger type must
 * be read in from the ELCR */

#define default_EISA_trigger(idx)	(EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
#define default_EISA_polarity(idx)	default_ISA_polarity(idx)

/* PCI interrupts are always polarity one level triggered,
 * when listed as conforming in the MP table. */

#define default_PCI_trigger(idx)	(1)
#define default_PCI_polarity(idx)	(1)

/* MCA interrupts are always polarity zero level triggered,
 * when listed as conforming in the MP table. */

#define default_MCA_trigger(idx)	(1)
#define default_MCA_polarity(idx)	default_ISA_polarity(idx)

static int MPBIOS_polarity(int idx)
{
	int bus = mp_irqs[idx].mp_srcbus;
	int polarity;

	/*
	 * Determine IRQ line polarity (high active or low active):
	 */
	switch (mp_irqs[idx].mp_irqflag & 3)
	{
		case 0: /* conforms, ie. bus-type dependent polarity */
			if (test_bit(bus, mp_bus_not_pci))
				polarity = default_ISA_polarity(idx);
			else
				polarity = default_PCI_polarity(idx);
			break;
		case 1: /* high active */
		{
			polarity = 0;
			break;
		}
		case 2: /* reserved */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
		case 3: /* low active */
		{
			polarity = 1;
			break;
		}
		default: /* invalid */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			polarity = 1;
			break;
		}
	}
	return polarity;
}

static int MPBIOS_trigger(int idx)
{
	int bus = mp_irqs[idx].mp_srcbus;
	int trigger;

	/*
	 * Determine IRQ trigger mode (edge or level sensitive):
	 */
	switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
	{
		case 0: /* conforms, ie. bus-type dependent */
			if (test_bit(bus, mp_bus_not_pci))
				trigger = default_ISA_trigger(idx);
			else
				trigger = default_PCI_trigger(idx);
#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
			switch (mp_bus_id_to_type[bus]) {
				case MP_BUS_ISA: /* ISA pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_EISA: /* EISA pin */
				{
					trigger = default_EISA_trigger(idx);
					break;
				}
				case MP_BUS_PCI: /* PCI pin */
				{
					/* set before the switch */
					break;
				}
				case MP_BUS_MCA: /* MCA pin */
				{
					trigger = default_MCA_trigger(idx);
					break;
				}
				default:
				{
					printk(KERN_WARNING "broken BIOS!!\n");
					trigger = 1;
					break;
				}
			}
#endif
			break;
		case 1: /* edge */
		{
			trigger = 0;
			break;
		}
		case 2: /* reserved */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			trigger = 1;
			break;
		}
		case 3: /* level */
		{
			trigger = 1;
			break;
		}
		default: /* invalid */
		{
			printk(KERN_WARNING "broken BIOS!!\n");
			trigger = 0;
			break;
		}
	}
	return trigger;
}

static inline int irq_polarity(int idx)
{
	return MPBIOS_polarity(idx);
}

static inline int irq_trigger(int idx)
{
	return MPBIOS_trigger(idx);
}

int (*ioapic_renumber_irq)(int ioapic, int irq);
static int pin_2_irq(int idx, int apic, int pin)
{
	int irq, i;
	int bus = mp_irqs[idx].mp_srcbus;

	/*
	 * Debugging check, we are in big trouble if this message pops up!
	 */
	if (mp_irqs[idx].mp_dstirq != pin)
		printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");

	if (test_bit(bus, mp_bus_not_pci)) {
		irq = mp_irqs[idx].mp_srcbusirq;
	} else {
		/*
		 * PCI IRQs are mapped in order
		 */
		i = irq = 0;
		while (i < apic)
			irq += nr_ioapic_registers[i++];
		irq += pin;
		/*
                 * For MPS mode, so far only needed by ES7000 platform
                 */
		if (ioapic_renumber_irq)
			irq = ioapic_renumber_irq(apic, irq);
	}

#ifdef CONFIG_X86_32
	/*
	 * PCI IRQ command line redirection. Yes, limits are hardcoded.
	 */
	if ((pin >= 16) && (pin <= 23)) {
		if (pirq_entries[pin-16] != -1) {
			if (!pirq_entries[pin-16]) {
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"disabling PIRQ%d\n", pin-16);
			} else {
				irq = pirq_entries[pin-16];
				apic_printk(APIC_VERBOSE, KERN_DEBUG
						"using PIRQ%d -> IRQ %d\n",
						pin-16, irq);
			}
		}
	}
#endif

	return irq;
}

void lock_vector_lock(void)
{
	/* Used to the online set of cpus does not change
	 * during assign_irq_vector.
	 */
	spin_lock(&vector_lock);
}

void unlock_vector_lock(void)
{
	spin_unlock(&vector_lock);
}

static int __assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask)
{
	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
	static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
	unsigned int old_vector;
	int cpu;

	if ((cfg->move_in_progress) || cfg->move_cleanup_count)
		return -EBUSY;

	/* Only try and allocate irqs on cpus that are present */
	cpus_and(mask, mask, cpu_online_map);

	old_vector = cfg->vector;
	if (old_vector) {
		cpumask_t tmp;
		cpus_and(tmp, cfg->domain, mask);
		if (!cpus_empty(tmp))
			return 0;
	}

	for_each_cpu_mask_nr(cpu, mask) {
		cpumask_t domain, new_mask;
		int new_cpu;
		int vector, offset;

		domain = vector_allocation_domain(cpu);
		cpus_and(new_mask, domain, cpu_online_map);

		vector = current_vector;
		offset = current_offset;
next:
		vector += 8;
		if (vector >= first_system_vector) {
			/* If we run out of vectors on large boxen, must share them. */
			offset = (offset + 1) % 8;
			vector = FIRST_DEVICE_VECTOR + offset;
		}
		if (unlikely(current_vector == vector))
			continue;
#ifdef CONFIG_X86_64
		if (vector == IA32_SYSCALL_VECTOR)
			goto next;
#else
		if (vector == SYSCALL_VECTOR)
			goto next;
#endif
		for_each_cpu_mask_nr(new_cpu, new_mask)
			if (per_cpu(vector_irq, new_cpu)[vector] != -1)
				goto next;
		/* Found one! */
		current_vector = vector;
		current_offset = offset;
		if (old_vector) {
			cfg->move_in_progress = 1;
			cfg->old_domain = cfg->domain;
		}
		for_each_cpu_mask_nr(new_cpu, new_mask)
			per_cpu(vector_irq, new_cpu)[vector] = irq;
		cfg->vector = vector;
		cfg->domain = domain;
		return 0;
	}
	return -ENOSPC;
}

static int assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask)
{
	int err;
	unsigned long flags;

	spin_lock_irqsave(&vector_lock, flags);
	err = __assign_irq_vector(irq, cfg, mask);
	spin_unlock_irqrestore(&vector_lock, flags);
	return err;
}

static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
{
	cpumask_t mask;
	int cpu, vector;

	BUG_ON(!cfg->vector);

	vector = cfg->vector;
	cpus_and(mask, cfg->domain, cpu_online_map);
	for_each_cpu_mask_nr(cpu, mask)
		per_cpu(vector_irq, cpu)[vector] = -1;

	cfg->vector = 0;
	cpus_clear(cfg->domain);

	if (likely(!cfg->move_in_progress))
		return;
	cpus_and(mask, cfg->old_domain, cpu_online_map);
	for_each_cpu_mask_nr(cpu, mask) {
		for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS;
								vector++) {
			if (per_cpu(vector_irq, cpu)[vector] != irq)
				continue;
			per_cpu(vector_irq, cpu)[vector] = -1;
			break;
		}
	}
	cfg->move_in_progress = 0;
}

void __setup_vector_irq(int cpu)
{
	/* Initialize vector_irq on a new cpu */
	/* This function must be called with vector_lock held */
	int irq, vector;
	struct irq_cfg *cfg;
	struct irq_desc *desc;

	/* Mark the inuse vectors */
	for_each_irq_desc(irq, desc) {
		if (!desc)
			continue;
		cfg = desc->chip_data;
		if (!cpu_isset(cpu, cfg->domain))
			continue;
		vector = cfg->vector;
		per_cpu(vector_irq, cpu)[vector] = irq;
	}
	/* Mark the free vectors */
	for (vector = 0; vector < NR_VECTORS; ++vector) {
		irq = per_cpu(vector_irq, cpu)[vector];
		if (irq < 0)
			continue;

		cfg = irq_cfg(irq);
		if (!cpu_isset(cpu, cfg->domain))
			per_cpu(vector_irq, cpu)[vector] = -1;
	}
}

static struct irq_chip ioapic_chip;
#ifdef CONFIG_INTR_REMAP
static struct irq_chip ir_ioapic_chip;
#endif

#define IOAPIC_AUTO     -1
#define IOAPIC_EDGE     0
#define IOAPIC_LEVEL    1

#ifdef CONFIG_X86_32
static inline int IO_APIC_irq_trigger(int irq)
{
	int apic, idx, pin;

	for (apic = 0; apic < nr_ioapics; apic++) {
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			idx = find_irq_entry(apic, pin, mp_INT);
			if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
				return irq_trigger(idx);
		}
	}
	/*
         * nonexistent IRQs are edge default
         */
	return 0;
}
#else
static inline int IO_APIC_irq_trigger(int irq)
{
	return 1;
}
#endif

static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger)
{

	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
	    trigger == IOAPIC_LEVEL)
		desc->status |= IRQ_LEVEL;
	else
		desc->status &= ~IRQ_LEVEL;

#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
		desc->status |= IRQ_MOVE_PCNTXT;
		if (trigger)
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_fasteoi_irq,
						     "fasteoi");
		else
			set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
						      handle_edge_irq, "edge");
		return;
	}
#endif
	if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
	    trigger == IOAPIC_LEVEL)
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
					      handle_fasteoi_irq,
					      "fasteoi");
	else
		set_irq_chip_and_handler_name(irq, &ioapic_chip,
					      handle_edge_irq, "edge");
}

static int setup_ioapic_entry(int apic, int irq,
			      struct IO_APIC_route_entry *entry,
			      unsigned int destination, int trigger,
			      int polarity, int vector)
{
	/*
	 * add it to the IO-APIC irq-routing table:
	 */
	memset(entry,0,sizeof(*entry));

#ifdef CONFIG_INTR_REMAP
	if (intr_remapping_enabled) {
		struct intel_iommu *iommu = map_ioapic_to_ir(apic);
		struct irte irte;
		struct IR_IO_APIC_route_entry *ir_entry =
			(struct IR_IO_APIC_route_entry *) entry;
		int index;

		if (!iommu)
			panic("No mapping iommu for ioapic %d\n", apic);

		index = alloc_irte(iommu, irq, 1);
		if (index < 0)
			panic("Failed to allocate IRTE for ioapic %d\n", apic);

		memset(&irte, 0, sizeof(irte));

		irte.present = 1;
		irte.dst_mode = INT_DEST_MODE;
		irte.trigger_mode = trigger;
		irte.dlvry_mode = INT_DELIVERY_MODE;
		irte.vector = vector;
		irte.dest_id = IRTE_DEST(destination);

		modify_irte(irq, &irte);

		ir_entry->index2 = (index >> 15) & 0x1;
		ir_entry->zero = 0;
		ir_entry->format = 1;
		ir_entry->index = (index & 0x7fff);
	} else
#endif
	{
		entry->delivery_mode = INT_DELIVERY_MODE;
		entry->dest_mode = INT_DEST_MODE;
		entry->dest = destination;
	}

	entry->mask = 0;				/* enable IRQ */
	entry->trigger = trigger;
	entry->polarity = polarity;
	entry->vector = vector;

	/* Mask level triggered irqs.
	 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
	 */
	if (trigger)
		entry->mask = 1;
	return 0;
}

static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, struct irq_desc *desc,
			      int trigger, int polarity)
{
	struct irq_cfg *cfg;
	struct IO_APIC_route_entry entry;
	cpumask_t mask;

	if (!IO_APIC_IRQ(irq))
		return;

	cfg = desc->chip_data;

	mask = TARGET_CPUS;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	cpus_and(mask, cfg->domain, mask);

	apic_printk(APIC_VERBOSE,KERN_DEBUG
		    "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
		    "IRQ %d Mode:%i Active:%i)\n",
		    apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
		    irq, trigger, polarity);


	if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
			       cpu_mask_to_apicid(mask), trigger, polarity,
			       cfg->vector)) {
		printk("Failed to setup ioapic entry for ioapic  %d, pin %d\n",
		       mp_ioapics[apic].mp_apicid, pin);
		__clear_irq_vector(irq, cfg);
		return;
	}

	ioapic_register_intr(irq, desc, trigger);
	if (irq < NR_IRQS_LEGACY)
		disable_8259A_irq(irq);

	ioapic_write_entry(apic, pin, entry);
}

static void __init setup_IO_APIC_irqs(void)
{
	int apic, pin, idx, irq;
	int notcon = 0;
	struct irq_desc *desc;
	struct irq_cfg *cfg;
	int cpu = boot_cpu_id;

	apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");

	for (apic = 0; apic < nr_ioapics; apic++) {
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {

			idx = find_irq_entry(apic, pin, mp_INT);
			if (idx == -1) {
				if (!notcon) {
					notcon = 1;
					apic_printk(APIC_VERBOSE,
						KERN_DEBUG " %d-%d",
						mp_ioapics[apic].mp_apicid,
						pin);
				} else
					apic_printk(APIC_VERBOSE, " %d-%d",
						mp_ioapics[apic].mp_apicid,
						pin);
				continue;
			}
			if (notcon) {
				apic_printk(APIC_VERBOSE,
					" (apicid-pin) not connected\n");
				notcon = 0;
			}

			irq = pin_2_irq(idx, apic, pin);
#ifdef CONFIG_X86_32
			if (multi_timer_check(apic, irq))
				continue;
#endif
			desc = irq_to_desc_alloc_cpu(irq, cpu);
			if (!desc) {
				printk(KERN_INFO "can not get irq_desc for %d\n", irq);
				continue;
			}
			cfg = desc->chip_data;
			add_pin_to_irq_cpu(cfg, cpu, apic, pin);

			setup_IO_APIC_irq(apic, pin, irq, desc,
					irq_trigger(idx), irq_polarity(idx));
		}
	}

	if (notcon)
		apic_printk(APIC_VERBOSE,
			" (apicid-pin) not connected\n");
}

/*
 * Set up the timer pin, possibly with the 8259A-master behind.
 */
static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
					int vector)
{
	struct IO_APIC_route_entry entry;

#ifdef CONFIG_INTR_REMAP
	if (intr_remapping_enabled)
		return;
#endif

	memset(&entry, 0, sizeof(entry));

	/*
	 * We use logical delivery to get the timer IRQ
	 * to the first CPU.
	 */
	entry.dest_mode = INT_DEST_MODE;
	entry.mask = 1;					/* mask IRQ now */
	entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
	entry.delivery_mode = INT_DELIVERY_MODE;
	entry.polarity = 0;
	entry.trigger = 0;
	entry.vector = vector;

	/*
	 * The timer IRQ doesn't have to know that behind the
	 * scene we may have a 8259A-master in AEOI mode ...
	 */
	set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");

	/*
	 * Add it to the IO-APIC irq-routing table:
	 */
	ioapic_write_entry(apic, pin, entry);
}


__apicdebuginit(void) print_IO_APIC(void)
{
	int apic, i;
	union IO_APIC_reg_00 reg_00;
	union IO_APIC_reg_01 reg_01;
	union IO_APIC_reg_02 reg_02;
	union IO_APIC_reg_03 reg_03;
	unsigned long flags;
	struct irq_cfg *cfg;
	struct irq_desc *desc;
	unsigned int irq;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
	for (i = 0; i < nr_ioapics; i++)
		printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
		       mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);

	/*
	 * We are a bit conservative about what we expect.  We have to
	 * know about every hardware change ASAP.
	 */
	printk(KERN_INFO "testing the IO APIC.......................\n");

	for (apic = 0; apic < nr_ioapics; apic++) {

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(apic, 0);
	reg_01.raw = io_apic_read(apic, 1);
	if (reg_01.bits.version >= 0x10)
		reg_02.raw = io_apic_read(apic, 2);
	if (reg_01.bits.version >= 0x20)
		reg_03.raw = io_apic_read(apic, 3);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	printk("\n");
	printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
	printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
	printk(KERN_DEBUG ".......    : physical APIC id: %02X\n", reg_00.bits.ID);
	printk(KERN_DEBUG ".......    : Delivery Type: %X\n", reg_00.bits.delivery_type);
	printk(KERN_DEBUG ".......    : LTS          : %X\n", reg_00.bits.LTS);

	printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
	printk(KERN_DEBUG ".......     : max redirection entries: %04X\n", reg_01.bits.entries);

	printk(KERN_DEBUG ".......     : PRQ implemented: %X\n", reg_01.bits.PRQ);
	printk(KERN_DEBUG ".......     : IO APIC version: %04X\n", reg_01.bits.version);

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
	 * but the value of reg_02 is read as the previous read register
	 * value, so ignore it if reg_02 == reg_01.
	 */
	if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
		printk(KERN_DEBUG ".......     : arbitration: %02X\n", reg_02.bits.arbitration);
	}

	/*
	 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
	 * or reg_03, but the value of reg_0[23] is read as the previous read
	 * register value, so ignore it if reg_03 == reg_0[12].
	 */
	if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
	    reg_03.raw != reg_01.raw) {
		printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
		printk(KERN_DEBUG ".......     : Boot DT    : %X\n", reg_03.bits.boot_DT);
	}

	printk(KERN_DEBUG ".... IRQ redirection table:\n");

	printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
			  " Stat Dmod Deli Vect:   \n");

	for (i = 0; i <= reg_01.bits.entries; i++) {
		struct IO_APIC_route_entry entry;

		entry = ioapic_read_entry(apic, i);

		printk(KERN_DEBUG " %02x %03X ",
			i,
			entry.dest
		);

		printk("%1d    %1d    %1d   %1d   %1d    %1d    %1d    %02X\n",
			entry.mask,
			entry.trigger,
			entry.irr,
			entry.polarity,
			entry.delivery_status,
			entry.dest_mode,
			entry.delivery_mode,
			entry.vector
		);
	}
	}
	printk(KERN_DEBUG "IRQ to pin mappings:\n");
	for_each_irq_desc(irq, desc) {
		struct irq_pin_list *entry;

		if (!desc)
			continue;
		cfg = desc->chip_data;
		entry = cfg->irq_2_pin;
		if (!entry)
			continue;
		printk(KERN_DEBUG "IRQ%d ", irq);
		for (;;) {
			printk("-> %d:%d", entry->apic, entry->pin);
			if (!entry->next)
				break;
			entry = entry->next;
		}
		printk("\n");
	}

	printk(KERN_INFO ".................................... done.\n");

	return;
}

__apicdebuginit(void) print_APIC_bitfield(int base)
{
	unsigned int v;
	int i, j;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "0123456789abcdef0123456789abcdef\n" KERN_DEBUG);
	for (i = 0; i < 8; i++) {
		v = apic_read(base + i*0x10);
		for (j = 0; j < 32; j++) {
			if (v & (1<<j))
				printk("1");
			else
				printk("0");
		}
		printk("\n");
	}
}

__apicdebuginit(void) print_local_APIC(void *dummy)
{
	unsigned int v, ver, maxlvt;
	u64 icr;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
		smp_processor_id(), hard_smp_processor_id());
	v = apic_read(APIC_ID);
	printk(KERN_INFO "... APIC ID:      %08x (%01x)\n", v, read_apic_id());
	v = apic_read(APIC_LVR);
	printk(KERN_INFO "... APIC VERSION: %08x\n", v);
	ver = GET_APIC_VERSION(v);
	maxlvt = lapic_get_maxlvt();

	v = apic_read(APIC_TASKPRI);
	printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);

	if (APIC_INTEGRATED(ver)) {                     /* !82489DX */
		if (!APIC_XAPIC(ver)) {
			v = apic_read(APIC_ARBPRI);
			printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
			       v & APIC_ARBPRI_MASK);
		}
		v = apic_read(APIC_PROCPRI);
		printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
	}

	/*
	 * Remote read supported only in the 82489DX and local APIC for
	 * Pentium processors.
	 */
	if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
		v = apic_read(APIC_RRR);
		printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
	}

	v = apic_read(APIC_LDR);
	printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
	if (!x2apic_enabled()) {
		v = apic_read(APIC_DFR);
		printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
	}
	v = apic_read(APIC_SPIV);
	printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);

	printk(KERN_DEBUG "... APIC ISR field:\n");
	print_APIC_bitfield(APIC_ISR);
	printk(KERN_DEBUG "... APIC TMR field:\n");
	print_APIC_bitfield(APIC_TMR);
	printk(KERN_DEBUG "... APIC IRR field:\n");
	print_APIC_bitfield(APIC_IRR);

	if (APIC_INTEGRATED(ver)) {             /* !82489DX */
		if (maxlvt > 3)         /* Due to the Pentium erratum 3AP. */
			apic_write(APIC_ESR, 0);

		v = apic_read(APIC_ESR);
		printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
	}

	icr = apic_icr_read();
	printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
	printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));

	v = apic_read(APIC_LVTT);
	printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);

	if (maxlvt > 3) {                       /* PC is LVT#4. */
		v = apic_read(APIC_LVTPC);
		printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
	}
	v = apic_read(APIC_LVT0);
	printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
	v = apic_read(APIC_LVT1);
	printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);

	if (maxlvt > 2) {			/* ERR is LVT#3. */
		v = apic_read(APIC_LVTERR);
		printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
	}

	v = apic_read(APIC_TMICT);
	printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
	v = apic_read(APIC_TMCCT);
	printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
	v = apic_read(APIC_TDCR);
	printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
	printk("\n");
}

__apicdebuginit(void) print_all_local_APICs(void)
{
	int cpu;

	preempt_disable();
	for_each_online_cpu(cpu)
		smp_call_function_single(cpu, print_local_APIC, NULL, 1);
	preempt_enable();
}

__apicdebuginit(void) print_PIC(void)
{
	unsigned int v;
	unsigned long flags;

	if (apic_verbosity == APIC_QUIET)
		return;

	printk(KERN_DEBUG "\nprinting PIC contents\n");

	spin_lock_irqsave(&i8259A_lock, flags);

	v = inb(0xa1) << 8 | inb(0x21);
	printk(KERN_DEBUG "... PIC  IMR: %04x\n", v);

	v = inb(0xa0) << 8 | inb(0x20);
	printk(KERN_DEBUG "... PIC  IRR: %04x\n", v);

	outb(0x0b,0xa0);
	outb(0x0b,0x20);
	v = inb(0xa0) << 8 | inb(0x20);
	outb(0x0a,0xa0);
	outb(0x0a,0x20);

	spin_unlock_irqrestore(&i8259A_lock, flags);

	printk(KERN_DEBUG "... PIC  ISR: %04x\n", v);

	v = inb(0x4d1) << 8 | inb(0x4d0);
	printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
}

__apicdebuginit(int) print_all_ICs(void)
{
	print_PIC();
	print_all_local_APICs();
	print_IO_APIC();

	return 0;
}

fs_initcall(print_all_ICs);


/* Where if anywhere is the i8259 connect in external int mode */
static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };

void __init enable_IO_APIC(void)
{
	union IO_APIC_reg_01 reg_01;
	int i8259_apic, i8259_pin;
	int apic;
	unsigned long flags;

#ifdef CONFIG_X86_32
	int i;
	if (!pirqs_enabled)
		for (i = 0; i < MAX_PIRQS; i++)
			pirq_entries[i] = -1;
#endif

	/*
	 * The number of IO-APIC IRQ registers (== #pins):
	 */
	for (apic = 0; apic < nr_ioapics; apic++) {
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_01.raw = io_apic_read(apic, 1);
		spin_unlock_irqrestore(&ioapic_lock, flags);
		nr_ioapic_registers[apic] = reg_01.bits.entries+1;
	}
	for(apic = 0; apic < nr_ioapics; apic++) {
		int pin;
		/* See if any of the pins is in ExtINT mode */
		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
			struct IO_APIC_route_entry entry;
			entry = ioapic_read_entry(apic, pin);

			/* If the interrupt line is enabled and in ExtInt mode
			 * I have found the pin where the i8259 is connected.
			 */
			if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
				ioapic_i8259.apic = apic;
				ioapic_i8259.pin  = pin;
				goto found_i8259;
			}
		}
	}
 found_i8259:
	/* Look to see what if the MP table has reported the ExtINT */
	/* If we could not find the appropriate pin by looking at the ioapic
	 * the i8259 probably is not connected the ioapic but give the
	 * mptable a chance anyway.
	 */
	i8259_pin  = find_isa_irq_pin(0, mp_ExtINT);
	i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
	/* Trust the MP table if nothing is setup in the hardware */
	if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
		printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
		ioapic_i8259.pin  = i8259_pin;
		ioapic_i8259.apic = i8259_apic;
	}
	/* Complain if the MP table and the hardware disagree */
	if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
		(i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
	{
		printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
	}

	/*
	 * Do not trust the IO-APIC being empty at bootup
	 */
	clear_IO_APIC();
}

/*
 * Not an __init, needed by the reboot code
 */
void disable_IO_APIC(void)
{
	/*
	 * Clear the IO-APIC before rebooting:
	 */
	clear_IO_APIC();

	/*
	 * If the i8259 is routed through an IOAPIC
	 * Put that IOAPIC in virtual wire mode
	 * so legacy interrupts can be delivered.
	 */
	if (ioapic_i8259.pin != -1) {
		struct IO_APIC_route_entry entry;

		memset(&entry, 0, sizeof(entry));
		entry.mask            = 0; /* Enabled */
		entry.trigger         = 0; /* Edge */
		entry.irr             = 0;
		entry.polarity        = 0; /* High */
		entry.delivery_status = 0;
		entry.dest_mode       = 0; /* Physical */
		entry.delivery_mode   = dest_ExtINT; /* ExtInt */
		entry.vector          = 0;
		entry.dest            = read_apic_id();

		/*
		 * Add it to the IO-APIC irq-routing table:
		 */
		ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
	}

	disconnect_bsp_APIC(ioapic_i8259.pin != -1);
}

#ifdef CONFIG_X86_32
/*
 * function to set the IO-APIC physical IDs based on the
 * values stored in the MPC table.
 *
 * by Matt Domsch <Matt_Domsch@dell.com>  Tue Dec 21 12:25:05 CST 1999
 */

static void __init setup_ioapic_ids_from_mpc(void)
{
	union IO_APIC_reg_00 reg_00;
	physid_mask_t phys_id_present_map;
	int apic;
	int i;
	unsigned char old_id;
	unsigned long flags;

	if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
		return;

	/*
	 * Don't check I/O APIC IDs for xAPIC systems.  They have
	 * no meaning without the serial APIC bus.
	 */
	if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		|| APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
		return;
	/*
	 * This is broken; anything with a real cpu count has to
	 * circumvent this idiocy regardless.
	 */
	phys_id_present_map = ioapic_phys_id_map(phys_cpu_present_map);

	/*
	 * Set the IOAPIC ID to the value stored in the MPC table.
	 */
	for (apic = 0; apic < nr_ioapics; apic++) {

		/* Read the register 0 value */
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_00.raw = io_apic_read(apic, 0);
		spin_unlock_irqrestore(&ioapic_lock, flags);

		old_id = mp_ioapics[apic].mp_apicid;

		if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
				apic, mp_ioapics[apic].mp_apicid);
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				reg_00.bits.ID);
			mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
		}

		/*
		 * Sanity check, is the ID really free? Every APIC in a
		 * system must have a unique ID or we get lots of nice
		 * 'stuck on smp_invalidate_needed IPI wait' messages.
		 */
		if (check_apicid_used(phys_id_present_map,
					mp_ioapics[apic].mp_apicid)) {
			printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
				apic, mp_ioapics[apic].mp_apicid);
			for (i = 0; i < get_physical_broadcast(); i++)
				if (!physid_isset(i, phys_id_present_map))
					break;
			if (i >= get_physical_broadcast())
				panic("Max APIC ID exceeded!\n");
			printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
				i);
			physid_set(i, phys_id_present_map);
			mp_ioapics[apic].mp_apicid = i;
		} else {
			physid_mask_t tmp;
			tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
			apic_printk(APIC_VERBOSE, "Setting %d in the "
					"phys_id_present_map\n",
					mp_ioapics[apic].mp_apicid);
			physids_or(phys_id_present_map, phys_id_present_map, tmp);
		}


		/*
		 * We need to adjust the IRQ routing table
		 * if the ID changed.
		 */
		if (old_id != mp_ioapics[apic].mp_apicid)
			for (i = 0; i < mp_irq_entries; i++)
				if (mp_irqs[i].mp_dstapic == old_id)
					mp_irqs[i].mp_dstapic
						= mp_ioapics[apic].mp_apicid;

		/*
		 * Read the right value from the MPC table and
		 * write it into the ID register.
		 */
		apic_printk(APIC_VERBOSE, KERN_INFO
			"...changing IO-APIC physical APIC ID to %d ...",
			mp_ioapics[apic].mp_apicid);

		reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
		spin_lock_irqsave(&ioapic_lock, flags);
		io_apic_write(apic, 0, reg_00.raw);
		spin_unlock_irqrestore(&ioapic_lock, flags);

		/*
		 * Sanity check
		 */
		spin_lock_irqsave(&ioapic_lock, flags);
		reg_00.raw = io_apic_read(apic, 0);
		spin_unlock_irqrestore(&ioapic_lock, flags);
		if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
			printk("could not set ID!\n");
		else
			apic_printk(APIC_VERBOSE, " ok.\n");
	}
}
#endif

int no_timer_check __initdata;

static int __init notimercheck(char *s)
{
	no_timer_check = 1;
	return 1;
}
__setup("no_timer_check", notimercheck);

/*
 * There is a nasty bug in some older SMP boards, their mptable lies
 * about the timer IRQ. We do the following to work around the situation:
 *
 *	- timer IRQ defaults to IO-APIC IRQ
 *	- if this function detects that timer IRQs are defunct, then we fall
 *	  back to ISA timer IRQs
 */
static int __init timer_irq_works(void)
{
	unsigned long t1 = jiffies;
	unsigned long flags;

	if (no_timer_check)
		return 1;

	local_save_flags(flags);
	local_irq_enable();
	/* Let ten ticks pass... */
	mdelay((10 * 1000) / HZ);
	local_irq_restore(flags);

	/*
	 * Expect a few ticks at least, to be sure some possible
	 * glue logic does not lock up after one or two first
	 * ticks in a non-ExtINT mode.  Also the local APIC
	 * might have cached one ExtINT interrupt.  Finally, at
	 * least one tick may be lost due to delays.
	 */

	/* jiffies wrap? */
	if (time_after(jiffies, t1 + 4))
		return 1;
	return 0;
}

/*
 * In the SMP+IOAPIC case it might happen that there are an unspecified
 * number of pending IRQ events unhandled. These cases are very rare,
 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
 * better to do it this way as thus we do not have to be aware of
 * 'pending' interrupts in the IRQ path, except at this point.
 */
/*
 * Edge triggered needs to resend any interrupt
 * that was delayed but this is now handled in the device
 * independent code.
 */

/*
 * Starting up a edge-triggered IO-APIC interrupt is
 * nasty - we need to make sure that we get the edge.
 * If it is already asserted for some reason, we need
 * return 1 to indicate that is was pending.
 *
 * This is not complete - we should be able to fake
 * an edge even if it isn't on the 8259A...
 */

static unsigned int startup_ioapic_irq(unsigned int irq)
{
	int was_pending = 0;
	unsigned long flags;
	struct irq_cfg *cfg;

	spin_lock_irqsave(&ioapic_lock, flags);
	if (irq < NR_IRQS_LEGACY) {
		disable_8259A_irq(irq);
		if (i8259A_irq_pending(irq))
			was_pending = 1;
	}
	cfg = irq_cfg(irq);
	__unmask_IO_APIC_irq(cfg);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return was_pending;
}

#ifdef CONFIG_X86_64
static int ioapic_retrigger_irq(unsigned int irq)
{

	struct irq_cfg *cfg = irq_cfg(irq);
	unsigned long flags;

	spin_lock_irqsave(&vector_lock, flags);
	send_IPI_mask(cpumask_of_cpu(first_cpu(cfg->domain)), cfg->vector);
	spin_unlock_irqrestore(&vector_lock, flags);

	return 1;
}
#else
static int ioapic_retrigger_irq(unsigned int irq)
{
	send_IPI_self(irq_cfg(irq)->vector);

	return 1;
}
#endif

/*
 * Level and edge triggered IO-APIC interrupts need different handling,
 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
 * handled with the level-triggered descriptor, but that one has slightly
 * more overhead. Level-triggered interrupts cannot be handled with the
 * edge-triggered handler, without risking IRQ storms and other ugly
 * races.
 */

#ifdef CONFIG_SMP

#ifdef CONFIG_INTR_REMAP
static void ir_irq_migration(struct work_struct *work);

static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);

/*
 * Migrate the IO-APIC irq in the presence of intr-remapping.
 *
 * For edge triggered, irq migration is a simple atomic update(of vector
 * and cpu destination) of IRTE and flush the hardware cache.
 *
 * For level triggered, we need to modify the io-apic RTE aswell with the update
 * vector information, along with modifying IRTE with vector and destination.
 * So irq migration for level triggered is little  bit more complex compared to
 * edge triggered migration. But the good news is, we use the same algorithm
 * for level triggered migration as we have today, only difference being,
 * we now initiate the irq migration from process context instead of the
 * interrupt context.
 *
 * In future, when we do a directed EOI (combined with cpu EOI broadcast
 * suppression) to the IO-APIC, level triggered irq migration will also be
 * as simple as edge triggered migration and we can do the irq migration
 * with a simple atomic update to IO-APIC RTE.
 */
static void migrate_ioapic_irq_desc(struct irq_desc *desc, cpumask_t mask)
{
	struct irq_cfg *cfg;
	cpumask_t tmp, cleanup_mask;
	struct irte irte;
	int modify_ioapic_rte;
	unsigned int dest;
	unsigned long flags;
	unsigned int irq;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	irq = desc->irq;
	if (get_irte(irq, &irte))
		return;

	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	set_extra_move_desc(desc, mask);

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	modify_ioapic_rte = desc->status & IRQ_LEVEL;
	if (modify_ioapic_rte) {
		spin_lock_irqsave(&ioapic_lock, flags);
		__target_IO_APIC_irq(irq, dest, cfg);
		spin_unlock_irqrestore(&ioapic_lock, flags);
	}

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * Modified the IRTE and flushes the Interrupt entry cache.
	 */
	modify_irte(irq, &irte);

	if (cfg->move_in_progress) {
		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}

	desc->affinity = mask;
}

static int migrate_irq_remapped_level_desc(struct irq_desc *desc)
{
	int ret = -1;
	struct irq_cfg *cfg = desc->chip_data;

	mask_IO_APIC_irq_desc(desc);

	if (io_apic_level_ack_pending(cfg)) {
		/*
		 * Interrupt in progress. Migrating irq now will change the
		 * vector information in the IO-APIC RTE and that will confuse
		 * the EOI broadcast performed by cpu.
		 * So, delay the irq migration to the next instance.
		 */
		schedule_delayed_work(&ir_migration_work, 1);
		goto unmask;
	}

	/* everthing is clear. we have right of way */
	migrate_ioapic_irq_desc(desc, desc->pending_mask);

	ret = 0;
	desc->status &= ~IRQ_MOVE_PENDING;
	cpus_clear(desc->pending_mask);

unmask:
	unmask_IO_APIC_irq_desc(desc);

	return ret;
}

static void ir_irq_migration(struct work_struct *work)
{
	unsigned int irq;
	struct irq_desc *desc;

	for_each_irq_desc(irq, desc) {
		if (!desc)
			continue;

		if (desc->status & IRQ_MOVE_PENDING) {
			unsigned long flags;

			spin_lock_irqsave(&desc->lock, flags);
			if (!desc->chip->set_affinity ||
			    !(desc->status & IRQ_MOVE_PENDING)) {
				desc->status &= ~IRQ_MOVE_PENDING;
				spin_unlock_irqrestore(&desc->lock, flags);
				continue;
			}

			desc->chip->set_affinity(irq, desc->pending_mask);
			spin_unlock_irqrestore(&desc->lock, flags);
		}
	}
}

/*
 * Migrates the IRQ destination in the process context.
 */
static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, cpumask_t mask)
{
	if (desc->status & IRQ_LEVEL) {
		desc->status |= IRQ_MOVE_PENDING;
		desc->pending_mask = mask;
		migrate_irq_remapped_level_desc(desc);
		return;
	}

	migrate_ioapic_irq_desc(desc, mask);
}
static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc = irq_to_desc(irq);

	set_ir_ioapic_affinity_irq_desc(desc, mask);
}
#endif

asmlinkage void smp_irq_move_cleanup_interrupt(void)
{
	unsigned vector, me;
	ack_APIC_irq();
#ifdef CONFIG_X86_64
	exit_idle();
#endif
	irq_enter();

	me = smp_processor_id();
	for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
		unsigned int irq;
		struct irq_desc *desc;
		struct irq_cfg *cfg;
		irq = __get_cpu_var(vector_irq)[vector];

		if (irq == -1)
			continue;

		desc = irq_to_desc(irq);
		if (!desc)
			continue;

		cfg = irq_cfg(irq);
		spin_lock(&desc->lock);
		if (!cfg->move_cleanup_count)
			goto unlock;

		if ((vector == cfg->vector) && cpu_isset(me, cfg->domain))
			goto unlock;

		__get_cpu_var(vector_irq)[vector] = -1;
		cfg->move_cleanup_count--;
unlock:
		spin_unlock(&desc->lock);
	}

	irq_exit();
}

static void irq_complete_move(struct irq_desc **descp)
{
	struct irq_desc *desc = *descp;
	struct irq_cfg *cfg = desc->chip_data;
	unsigned vector, me;

	if (likely(!cfg->move_in_progress))
		return;

	vector = ~get_irq_regs()->orig_ax;
	me = smp_processor_id();
	if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
		cpumask_t cleanup_mask;

		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}
}
#else
static inline void irq_complete_move(struct irq_desc **descp) {}
#endif

#ifdef CONFIG_INTR_REMAP
static void ack_x2apic_level(unsigned int irq)
{
	ack_x2APIC_irq();
}

static void ack_x2apic_edge(unsigned int irq)
{
	ack_x2APIC_irq();
}

#endif

static void ack_apic_edge(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

	irq_complete_move(&desc);
	move_native_irq(irq);
	ack_APIC_irq();
}

atomic_t irq_mis_count;

static void ack_apic_level(unsigned int irq)
{
	struct irq_desc *desc = irq_to_desc(irq);

#ifdef CONFIG_X86_32
	unsigned long v;
	int i;
#endif
	struct irq_cfg *cfg;
	int do_unmask_irq = 0;

	irq_complete_move(&desc);
#ifdef CONFIG_GENERIC_PENDING_IRQ
	/* If we are moving the irq we need to mask it */
	if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
		do_unmask_irq = 1;
		mask_IO_APIC_irq_desc(desc);
	}
#endif

#ifdef CONFIG_X86_32
	/*
	* It appears there is an erratum which affects at least version 0x11
	* of I/O APIC (that's the 82093AA and cores integrated into various
	* chipsets).  Under certain conditions a level-triggered interrupt is
	* erroneously delivered as edge-triggered one but the respective IRR
	* bit gets set nevertheless.  As a result the I/O unit expects an EOI
	* message but it will never arrive and further interrupts are blocked
	* from the source.  The exact reason is so far unknown, but the
	* phenomenon was observed when two consecutive interrupt requests
	* from a given source get delivered to the same CPU and the source is
	* temporarily disabled in between.
	*
	* A workaround is to simulate an EOI message manually.  We achieve it
	* by setting the trigger mode to edge and then to level when the edge
	* trigger mode gets detected in the TMR of a local APIC for a
	* level-triggered interrupt.  We mask the source for the time of the
	* operation to prevent an edge-triggered interrupt escaping meanwhile.
	* The idea is from Manfred Spraul.  --macro
	*/
	cfg = desc->chip_data;
	i = cfg->vector;

	v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
#endif

	/*
	 * We must acknowledge the irq before we move it or the acknowledge will
	 * not propagate properly.
	 */
	ack_APIC_irq();

	/* Now we can move and renable the irq */
	if (unlikely(do_unmask_irq)) {
		/* Only migrate the irq if the ack has been received.
		 *
		 * On rare occasions the broadcast level triggered ack gets
		 * delayed going to ioapics, and if we reprogram the
		 * vector while Remote IRR is still set the irq will never
		 * fire again.
		 *
		 * To prevent this scenario we read the Remote IRR bit
		 * of the ioapic.  This has two effects.
		 * - On any sane system the read of the ioapic will
		 *   flush writes (and acks) going to the ioapic from
		 *   this cpu.
		 * - We get to see if the ACK has actually been delivered.
		 *
		 * Based on failed experiments of reprogramming the
		 * ioapic entry from outside of irq context starting
		 * with masking the ioapic entry and then polling until
		 * Remote IRR was clear before reprogramming the
		 * ioapic I don't trust the Remote IRR bit to be
		 * completey accurate.
		 *
		 * However there appears to be no other way to plug
		 * this race, so if the Remote IRR bit is not
		 * accurate and is causing problems then it is a hardware bug
		 * and you can go talk to the chipset vendor about it.
		 */
		cfg = desc->chip_data;
		if (!io_apic_level_ack_pending(cfg))
			move_masked_irq(irq);
		unmask_IO_APIC_irq_desc(desc);
	}

#ifdef CONFIG_X86_32
	if (!(v & (1 << (i & 0x1f)))) {
		atomic_inc(&irq_mis_count);
		spin_lock(&ioapic_lock);
		__mask_and_edge_IO_APIC_irq(cfg);
		__unmask_and_level_IO_APIC_irq(cfg);
		spin_unlock(&ioapic_lock);
	}
#endif
}

static struct irq_chip ioapic_chip __read_mostly = {
	.name		= "IO-APIC",
	.startup	= startup_ioapic_irq,
	.mask		= mask_IO_APIC_irq,
	.unmask		= unmask_IO_APIC_irq,
	.ack		= ack_apic_edge,
	.eoi		= ack_apic_level,
#ifdef CONFIG_SMP
	.set_affinity	= set_ioapic_affinity_irq,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

#ifdef CONFIG_INTR_REMAP
static struct irq_chip ir_ioapic_chip __read_mostly = {
	.name		= "IR-IO-APIC",
	.startup	= startup_ioapic_irq,
	.mask		= mask_IO_APIC_irq,
	.unmask		= unmask_IO_APIC_irq,
	.ack		= ack_x2apic_edge,
	.eoi		= ack_x2apic_level,
#ifdef CONFIG_SMP
	.set_affinity	= set_ir_ioapic_affinity_irq,
#endif
	.retrigger	= ioapic_retrigger_irq,
};
#endif

static inline void init_IO_APIC_traps(void)
{
	int irq;
	struct irq_desc *desc;
	struct irq_cfg *cfg;

	/*
	 * NOTE! The local APIC isn't very good at handling
	 * multiple interrupts at the same interrupt level.
	 * As the interrupt level is determined by taking the
	 * vector number and shifting that right by 4, we
	 * want to spread these out a bit so that they don't
	 * all fall in the same interrupt level.
	 *
	 * Also, we've got to be careful not to trash gate
	 * 0x80, because int 0x80 is hm, kind of importantish. ;)
	 */
	for_each_irq_desc(irq, desc) {
		if (!desc)
			continue;

		cfg = desc->chip_data;
		if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
			/*
			 * Hmm.. We don't have an entry for this,
			 * so default to an old-fashioned 8259
			 * interrupt if we can..
			 */
			if (irq < NR_IRQS_LEGACY)
				make_8259A_irq(irq);
			else
				/* Strange. Oh, well.. */
				desc->chip = &no_irq_chip;
		}
	}
}

/*
 * The local APIC irq-chip implementation:
 */

static void mask_lapic_irq(unsigned int irq)
{
	unsigned long v;

	v = apic_read(APIC_LVT0);
	apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
}

static void unmask_lapic_irq(unsigned int irq)
{
	unsigned long v;

	v = apic_read(APIC_LVT0);
	apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
}

static void ack_lapic_irq(unsigned int irq)
{
	ack_APIC_irq();
}

static struct irq_chip lapic_chip __read_mostly = {
	.name		= "local-APIC",
	.mask		= mask_lapic_irq,
	.unmask		= unmask_lapic_irq,
	.ack		= ack_lapic_irq,
};

static void lapic_register_intr(int irq, struct irq_desc *desc)
{
	desc->status &= ~IRQ_LEVEL;
	set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
				      "edge");
}

static void __init setup_nmi(void)
{
	/*
	 * Dirty trick to enable the NMI watchdog ...
	 * We put the 8259A master into AEOI mode and
	 * unmask on all local APICs LVT0 as NMI.
	 *
	 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
	 * is from Maciej W. Rozycki - so we do not have to EOI from
	 * the NMI handler or the timer interrupt.
	 */
	apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");

	enable_NMI_through_LVT0();

	apic_printk(APIC_VERBOSE, " done.\n");
}

/*
 * This looks a bit hackish but it's about the only one way of sending
 * a few INTA cycles to 8259As and any associated glue logic.  ICR does
 * not support the ExtINT mode, unfortunately.  We need to send these
 * cycles as some i82489DX-based boards have glue logic that keeps the
 * 8259A interrupt line asserted until INTA.  --macro
 */
static inline void __init unlock_ExtINT_logic(void)
{
	int apic, pin, i;
	struct IO_APIC_route_entry entry0, entry1;
	unsigned char save_control, save_freq_select;

	pin  = find_isa_irq_pin(8, mp_INT);
	if (pin == -1) {
		WARN_ON_ONCE(1);
		return;
	}
	apic = find_isa_irq_apic(8, mp_INT);
	if (apic == -1) {
		WARN_ON_ONCE(1);
		return;
	}

	entry0 = ioapic_read_entry(apic, pin);
	clear_IO_APIC_pin(apic, pin);

	memset(&entry1, 0, sizeof(entry1));

	entry1.dest_mode = 0;			/* physical delivery */
	entry1.mask = 0;			/* unmask IRQ now */
	entry1.dest = hard_smp_processor_id();
	entry1.delivery_mode = dest_ExtINT;
	entry1.polarity = entry0.polarity;
	entry1.trigger = 0;
	entry1.vector = 0;

	ioapic_write_entry(apic, pin, entry1);

	save_control = CMOS_READ(RTC_CONTROL);
	save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
	CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
		   RTC_FREQ_SELECT);
	CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);

	i = 100;
	while (i-- > 0) {
		mdelay(10);
		if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
			i -= 10;
	}

	CMOS_WRITE(save_control, RTC_CONTROL);
	CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
	clear_IO_APIC_pin(apic, pin);

	ioapic_write_entry(apic, pin, entry0);
}

static int disable_timer_pin_1 __initdata;
/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
static int __init disable_timer_pin_setup(char *arg)
{
	disable_timer_pin_1 = 1;
	return 0;
}
early_param("disable_timer_pin_1", disable_timer_pin_setup);

int timer_through_8259 __initdata;

/*
 * This code may look a bit paranoid, but it's supposed to cooperate with
 * a wide range of boards and BIOS bugs.  Fortunately only the timer IRQ
 * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
 * fanatically on his truly buggy board.
 *
 * FIXME: really need to revamp this for all platforms.
 */
static inline void __init check_timer(void)
{
	struct irq_desc *desc = irq_to_desc(0);
	struct irq_cfg *cfg = desc->chip_data;
	int cpu = boot_cpu_id;
	int apic1, pin1, apic2, pin2;
	unsigned long flags;
	unsigned int ver;
	int no_pin1 = 0;

	local_irq_save(flags);

	ver = apic_read(APIC_LVR);
	ver = GET_APIC_VERSION(ver);

	/*
	 * get/set the timer IRQ vector:
	 */
	disable_8259A_irq(0);
	assign_irq_vector(0, cfg, TARGET_CPUS);

	/*
	 * As IRQ0 is to be enabled in the 8259A, the virtual
	 * wire has to be disabled in the local APIC.  Also
	 * timer interrupts need to be acknowledged manually in
	 * the 8259A for the i82489DX when using the NMI
	 * watchdog as that APIC treats NMIs as level-triggered.
	 * The AEOI mode will finish them in the 8259A
	 * automatically.
	 */
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
	init_8259A(1);
#ifdef CONFIG_X86_32
	timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
#endif

	pin1  = find_isa_irq_pin(0, mp_INT);
	apic1 = find_isa_irq_apic(0, mp_INT);
	pin2  = ioapic_i8259.pin;
	apic2 = ioapic_i8259.apic;

	apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
		    "apic1=%d pin1=%d apic2=%d pin2=%d\n",
		    cfg->vector, apic1, pin1, apic2, pin2);

	/*
	 * Some BIOS writers are clueless and report the ExtINTA
	 * I/O APIC input from the cascaded 8259A as the timer
	 * interrupt input.  So just in case, if only one pin
	 * was found above, try it both directly and through the
	 * 8259A.
	 */
	if (pin1 == -1) {
#ifdef CONFIG_INTR_REMAP
		if (intr_remapping_enabled)
			panic("BIOS bug: timer not connected to IO-APIC");
#endif
		pin1 = pin2;
		apic1 = apic2;
		no_pin1 = 1;
	} else if (pin2 == -1) {
		pin2 = pin1;
		apic2 = apic1;
	}

	if (pin1 != -1) {
		/*
		 * Ok, does IRQ0 through the IOAPIC work?
		 */
		if (no_pin1) {
			add_pin_to_irq_cpu(cfg, cpu, apic1, pin1);
			setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
		}
		unmask_IO_APIC_irq_desc(desc);
		if (timer_irq_works()) {
			if (nmi_watchdog == NMI_IO_APIC) {
				setup_nmi();
				enable_8259A_irq(0);
			}
			if (disable_timer_pin_1 > 0)
				clear_IO_APIC_pin(0, pin1);
			goto out;
		}
#ifdef CONFIG_INTR_REMAP
		if (intr_remapping_enabled)
			panic("timer doesn't work through Interrupt-remapped IO-APIC");
#endif
		clear_IO_APIC_pin(apic1, pin1);
		if (!no_pin1)
			apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
				    "8254 timer not connected to IO-APIC\n");

		apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
			    "(IRQ0) through the 8259A ...\n");
		apic_printk(APIC_QUIET, KERN_INFO
			    "..... (found apic %d pin %d) ...\n", apic2, pin2);
		/*
		 * legacy devices should be connected to IO APIC #0
		 */
		replace_pin_at_irq_cpu(cfg, cpu, apic1, pin1, apic2, pin2);
		setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
		unmask_IO_APIC_irq_desc(desc);
		enable_8259A_irq(0);
		if (timer_irq_works()) {
			apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
			timer_through_8259 = 1;
			if (nmi_watchdog == NMI_IO_APIC) {
				disable_8259A_irq(0);
				setup_nmi();
				enable_8259A_irq(0);
			}
			goto out;
		}
		/*
		 * Cleanup, just in case ...
		 */
		disable_8259A_irq(0);
		clear_IO_APIC_pin(apic2, pin2);
		apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
	}

	if (nmi_watchdog == NMI_IO_APIC) {
		apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
			    "through the IO-APIC - disabling NMI Watchdog!\n");
		nmi_watchdog = NMI_NONE;
	}
#ifdef CONFIG_X86_32
	timer_ack = 0;
#endif

	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as Virtual Wire IRQ...\n");

	lapic_register_intr(0, desc);
	apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector);	/* Fixed mode */
	enable_8259A_irq(0);

	if (timer_irq_works()) {
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
		goto out;
	}
	disable_8259A_irq(0);
	apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
	apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");

	apic_printk(APIC_QUIET, KERN_INFO
		    "...trying to set up timer as ExtINT IRQ...\n");

	init_8259A(0);
	make_8259A_irq(0);
	apic_write(APIC_LVT0, APIC_DM_EXTINT);

	unlock_ExtINT_logic();

	if (timer_irq_works()) {
		apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
		goto out;
	}
	apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
	panic("IO-APIC + timer doesn't work!  Boot with apic=debug and send a "
		"report.  Then try booting with the 'noapic' option.\n");
out:
	local_irq_restore(flags);
}

/*
 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
 * to devices.  However there may be an I/O APIC pin available for
 * this interrupt regardless.  The pin may be left unconnected, but
 * typically it will be reused as an ExtINT cascade interrupt for
 * the master 8259A.  In the MPS case such a pin will normally be
 * reported as an ExtINT interrupt in the MP table.  With ACPI
 * there is no provision for ExtINT interrupts, and in the absence
 * of an override it would be treated as an ordinary ISA I/O APIC
 * interrupt, that is edge-triggered and unmasked by default.  We
 * used to do this, but it caused problems on some systems because
 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
 * the same ExtINT cascade interrupt to drive the local APIC of the
 * bootstrap processor.  Therefore we refrain from routing IRQ2 to
 * the I/O APIC in all cases now.  No actual device should request
 * it anyway.  --macro
 */
#define PIC_IRQS	(1 << PIC_CASCADE_IR)

void __init setup_IO_APIC(void)
{

#ifdef CONFIG_X86_32
	enable_IO_APIC();
#else
	/*
	 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
	 */
#endif

	io_apic_irqs = ~PIC_IRQS;

	apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
	/*
         * Set up IO-APIC IRQ routing.
         */
#ifdef CONFIG_X86_32
	if (!acpi_ioapic)
		setup_ioapic_ids_from_mpc();
#endif
	sync_Arb_IDs();
	setup_IO_APIC_irqs();
	init_IO_APIC_traps();
	check_timer();
}

/*
 *      Called after all the initialization is done. If we didnt find any
 *      APIC bugs then we can allow the modify fast path
 */

static int __init io_apic_bug_finalize(void)
{
	if (sis_apic_bug == -1)
		sis_apic_bug = 0;
	return 0;
}

late_initcall(io_apic_bug_finalize);

struct sysfs_ioapic_data {
	struct sys_device dev;
	struct IO_APIC_route_entry entry[0];
};
static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];

static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	int i;

	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;
	for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
		*entry = ioapic_read_entry(dev->id, i);

	return 0;
}

static int ioapic_resume(struct sys_device *dev)
{
	struct IO_APIC_route_entry *entry;
	struct sysfs_ioapic_data *data;
	unsigned long flags;
	union IO_APIC_reg_00 reg_00;
	int i;

	data = container_of(dev, struct sysfs_ioapic_data, dev);
	entry = data->entry;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(dev->id, 0);
	if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
		reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
		io_apic_write(dev->id, 0, reg_00.raw);
	}
	spin_unlock_irqrestore(&ioapic_lock, flags);
	for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
		ioapic_write_entry(dev->id, i, entry[i]);

	return 0;
}

static struct sysdev_class ioapic_sysdev_class = {
	.name = "ioapic",
	.suspend = ioapic_suspend,
	.resume = ioapic_resume,
};

static int __init ioapic_init_sysfs(void)
{
	struct sys_device * dev;
	int i, size, error;

	error = sysdev_class_register(&ioapic_sysdev_class);
	if (error)
		return error;

	for (i = 0; i < nr_ioapics; i++ ) {
		size = sizeof(struct sys_device) + nr_ioapic_registers[i]
			* sizeof(struct IO_APIC_route_entry);
		mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
		if (!mp_ioapic_data[i]) {
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
		dev = &mp_ioapic_data[i]->dev;
		dev->id = i;
		dev->cls = &ioapic_sysdev_class;
		error = sysdev_register(dev);
		if (error) {
			kfree(mp_ioapic_data[i]);
			mp_ioapic_data[i] = NULL;
			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
			continue;
		}
	}

	return 0;
}

device_initcall(ioapic_init_sysfs);

/*
 * Dynamic irq allocate and deallocation
 */
unsigned int create_irq_nr(unsigned int irq_want)
{
	/* Allocate an unused irq */
	unsigned int irq;
	unsigned int new;
	unsigned long flags;
	struct irq_cfg *cfg_new = NULL;
	int cpu = boot_cpu_id;
	struct irq_desc *desc_new = NULL;

	irq = 0;
	spin_lock_irqsave(&vector_lock, flags);
	for (new = irq_want; new < NR_IRQS; new++) {
		if (platform_legacy_irq(new))
			continue;

		desc_new = irq_to_desc_alloc_cpu(new, cpu);
		if (!desc_new) {
			printk(KERN_INFO "can not get irq_desc for %d\n", new);
			continue;
		}
		cfg_new = desc_new->chip_data;

		if (cfg_new->vector != 0)
			continue;
		if (__assign_irq_vector(new, cfg_new, TARGET_CPUS) == 0)
			irq = new;
		break;
	}
	spin_unlock_irqrestore(&vector_lock, flags);

	if (irq > 0) {
		dynamic_irq_init(irq);
		/* restore it, in case dynamic_irq_init clear it */
		if (desc_new)
			desc_new->chip_data = cfg_new;
	}
	return irq;
}

static int nr_irqs_gsi = NR_IRQS_LEGACY;
int create_irq(void)
{
	unsigned int irq_want;
	int irq;

	irq_want = nr_irqs_gsi;
	irq = create_irq_nr(irq_want);

	if (irq == 0)
		irq = -1;

	return irq;
}

void destroy_irq(unsigned int irq)
{
	unsigned long flags;
	struct irq_cfg *cfg;
	struct irq_desc *desc;

	/* store it, in case dynamic_irq_cleanup clear it */
	desc = irq_to_desc(irq);
	cfg = desc->chip_data;
	dynamic_irq_cleanup(irq);
	/* connect back irq_cfg */
	if (desc)
		desc->chip_data = cfg;

#ifdef CONFIG_INTR_REMAP
	free_irte(irq);
#endif
	spin_lock_irqsave(&vector_lock, flags);
	__clear_irq_vector(irq, cfg);
	spin_unlock_irqrestore(&vector_lock, flags);
}

/*
 * MSI message composition
 */
#ifdef CONFIG_PCI_MSI
static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
{
	struct irq_cfg *cfg;
	int err;
	unsigned dest;
	cpumask_t tmp;

	cfg = irq_cfg(irq);
	tmp = TARGET_CPUS;
	err = assign_irq_vector(irq, cfg, tmp);
	if (err)
		return err;

	cpus_and(tmp, cfg->domain, tmp);
	dest = cpu_mask_to_apicid(tmp);

#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
		struct irte irte;
		int ir_index;
		u16 sub_handle;

		ir_index = map_irq_to_irte_handle(irq, &sub_handle);
		BUG_ON(ir_index == -1);

		memset (&irte, 0, sizeof(irte));

		irte.present = 1;
		irte.dst_mode = INT_DEST_MODE;
		irte.trigger_mode = 0; /* edge */
		irte.dlvry_mode = INT_DELIVERY_MODE;
		irte.vector = cfg->vector;
		irte.dest_id = IRTE_DEST(dest);

		modify_irte(irq, &irte);

		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->data = sub_handle;
		msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
				  MSI_ADDR_IR_SHV |
				  MSI_ADDR_IR_INDEX1(ir_index) |
				  MSI_ADDR_IR_INDEX2(ir_index);
	} else
#endif
	{
		msg->address_hi = MSI_ADDR_BASE_HI;
		msg->address_lo =
			MSI_ADDR_BASE_LO |
			((INT_DEST_MODE == 0) ?
				MSI_ADDR_DEST_MODE_PHYSICAL:
				MSI_ADDR_DEST_MODE_LOGICAL) |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				MSI_ADDR_REDIRECTION_CPU:
				MSI_ADDR_REDIRECTION_LOWPRI) |
			MSI_ADDR_DEST_ID(dest);

		msg->data =
			MSI_DATA_TRIGGER_EDGE |
			MSI_DATA_LEVEL_ASSERT |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				MSI_DATA_DELIVERY_FIXED:
				MSI_DATA_DELIVERY_LOWPRI) |
			MSI_DATA_VECTOR(cfg->vector);
	}
	return err;
}

#ifdef CONFIG_SMP
static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc = irq_to_desc(irq);
	struct irq_cfg *cfg;
	struct msi_msg msg;
	unsigned int dest;
	cpumask_t tmp;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	set_extra_move_desc(desc, mask);

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	read_msi_msg_desc(desc, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	write_msi_msg_desc(desc, &msg);
	desc->affinity = mask;
}
#ifdef CONFIG_INTR_REMAP
/*
 * Migrate the MSI irq to another cpumask. This migration is
 * done in the process context using interrupt-remapping hardware.
 */
static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc = irq_to_desc(irq);
	struct irq_cfg *cfg;
	unsigned int dest;
	cpumask_t tmp, cleanup_mask;
	struct irte irte;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	if (get_irte(irq, &irte))
		return;

	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	set_extra_move_desc(desc, mask);

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	irte.vector = cfg->vector;
	irte.dest_id = IRTE_DEST(dest);

	/*
	 * atomically update the IRTE with the new destination and vector.
	 */
	modify_irte(irq, &irte);

	/*
	 * After this point, all the interrupts will start arriving
	 * at the new destination. So, time to cleanup the previous
	 * vector allocation.
	 */
	if (cfg->move_in_progress) {
		cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
		cfg->move_cleanup_count = cpus_weight(cleanup_mask);
		send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
		cfg->move_in_progress = 0;
	}

	desc->affinity = mask;
}

#endif
#endif /* CONFIG_SMP */

/*
 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
 * which implement the MSI or MSI-X Capability Structure.
 */
static struct irq_chip msi_chip = {
	.name		= "PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
	.ack		= ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity	= set_msi_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

#ifdef CONFIG_INTR_REMAP
static struct irq_chip msi_ir_chip = {
	.name		= "IR-PCI-MSI",
	.unmask		= unmask_msi_irq,
	.mask		= mask_msi_irq,
	.ack		= ack_x2apic_edge,
#ifdef CONFIG_SMP
	.set_affinity	= ir_set_msi_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

/*
 * Map the PCI dev to the corresponding remapping hardware unit
 * and allocate 'nvec' consecutive interrupt-remapping table entries
 * in it.
 */
static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
{
	struct intel_iommu *iommu;
	int index;

	iommu = map_dev_to_ir(dev);
	if (!iommu) {
		printk(KERN_ERR
		       "Unable to map PCI %s to iommu\n", pci_name(dev));
		return -ENOENT;
	}

	index = alloc_irte(iommu, irq, nvec);
	if (index < 0) {
		printk(KERN_ERR
		       "Unable to allocate %d IRTE for PCI %s\n", nvec,
		       pci_name(dev));
		return -ENOSPC;
	}
	return index;
}
#endif

static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
{
	int ret;
	struct msi_msg msg;

	ret = msi_compose_msg(dev, irq, &msg);
	if (ret < 0)
		return ret;

	set_irq_msi(irq, msidesc);
	write_msi_msg(irq, &msg);

#ifdef CONFIG_INTR_REMAP
	if (irq_remapped(irq)) {
		struct irq_desc *desc = irq_to_desc(irq);
		/*
		 * irq migration in process context
		 */
		desc->status |= IRQ_MOVE_PCNTXT;
		set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
	} else
#endif
		set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");

	dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq);

	return 0;
}

int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc)
{
	unsigned int irq;
	int ret;
	unsigned int irq_want;

	irq_want = nr_irqs_gsi;
	irq = create_irq_nr(irq_want);
	if (irq == 0)
		return -1;

#ifdef CONFIG_INTR_REMAP
	if (!intr_remapping_enabled)
		goto no_ir;

	ret = msi_alloc_irte(dev, irq, 1);
	if (ret < 0)
		goto error;
no_ir:
#endif
	ret = setup_msi_irq(dev, msidesc, irq);
	if (ret < 0) {
		destroy_irq(irq);
		return ret;
	}
	return 0;

#ifdef CONFIG_INTR_REMAP
error:
	destroy_irq(irq);
	return ret;
#endif
}

int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
{
	unsigned int irq;
	int ret, sub_handle;
	struct msi_desc *msidesc;
	unsigned int irq_want;

#ifdef CONFIG_INTR_REMAP
	struct intel_iommu *iommu = 0;
	int index = 0;
#endif

	irq_want = nr_irqs_gsi;
	sub_handle = 0;
	list_for_each_entry(msidesc, &dev->msi_list, list) {
		irq = create_irq_nr(irq_want);
		irq_want++;
		if (irq == 0)
			return -1;
#ifdef CONFIG_INTR_REMAP
		if (!intr_remapping_enabled)
			goto no_ir;

		if (!sub_handle) {
			/*
			 * allocate the consecutive block of IRTE's
			 * for 'nvec'
			 */
			index = msi_alloc_irte(dev, irq, nvec);
			if (index < 0) {
				ret = index;
				goto error;
			}
		} else {
			iommu = map_dev_to_ir(dev);
			if (!iommu) {
				ret = -ENOENT;
				goto error;
			}
			/*
			 * setup the mapping between the irq and the IRTE
			 * base index, the sub_handle pointing to the
			 * appropriate interrupt remap table entry.
			 */
			set_irte_irq(irq, iommu, index, sub_handle);
		}
no_ir:
#endif
		ret = setup_msi_irq(dev, msidesc, irq);
		if (ret < 0)
			goto error;
		sub_handle++;
	}
	return 0;

error:
	destroy_irq(irq);
	return ret;
}

void arch_teardown_msi_irq(unsigned int irq)
{
	destroy_irq(irq);
}

#ifdef CONFIG_DMAR
#ifdef CONFIG_SMP
static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc = irq_to_desc(irq);
	struct irq_cfg *cfg;
	struct msi_msg msg;
	unsigned int dest;
	cpumask_t tmp;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	set_extra_move_desc(desc, mask);

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	dmar_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	dmar_msi_write(irq, &msg);
	desc->affinity = mask;
}

#endif /* CONFIG_SMP */

struct irq_chip dmar_msi_type = {
	.name = "DMAR_MSI",
	.unmask = dmar_msi_unmask,
	.mask = dmar_msi_mask,
	.ack = ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity = dmar_msi_set_affinity,
#endif
	.retrigger = ioapic_retrigger_irq,
};

int arch_setup_dmar_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;

	ret = msi_compose_msg(NULL, irq, &msg);
	if (ret < 0)
		return ret;
	dmar_msi_write(irq, &msg);
	set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
		"edge");
	return 0;
}
#endif

#ifdef CONFIG_HPET_TIMER

#ifdef CONFIG_SMP
static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc = irq_to_desc(irq);
	struct irq_cfg *cfg;
	struct msi_msg msg;
	unsigned int dest;
	cpumask_t tmp;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	set_extra_move_desc(desc, mask);

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	hpet_msi_read(irq, &msg);

	msg.data &= ~MSI_DATA_VECTOR_MASK;
	msg.data |= MSI_DATA_VECTOR(cfg->vector);
	msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
	msg.address_lo |= MSI_ADDR_DEST_ID(dest);

	hpet_msi_write(irq, &msg);
	desc->affinity = mask;
}

#endif /* CONFIG_SMP */

struct irq_chip hpet_msi_type = {
	.name = "HPET_MSI",
	.unmask = hpet_msi_unmask,
	.mask = hpet_msi_mask,
	.ack = ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity = hpet_msi_set_affinity,
#endif
	.retrigger = ioapic_retrigger_irq,
};

int arch_setup_hpet_msi(unsigned int irq)
{
	int ret;
	struct msi_msg msg;

	ret = msi_compose_msg(NULL, irq, &msg);
	if (ret < 0)
		return ret;

	hpet_msi_write(irq, &msg);
	set_irq_chip_and_handler_name(irq, &hpet_msi_type, handle_edge_irq,
		"edge");

	return 0;
}
#endif

#endif /* CONFIG_PCI_MSI */
/*
 * Hypertransport interrupt support
 */
#ifdef CONFIG_HT_IRQ

#ifdef CONFIG_SMP

static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
{
	struct ht_irq_msg msg;
	fetch_ht_irq_msg(irq, &msg);

	msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
	msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);

	msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
	msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);

	write_ht_irq_msg(irq, &msg);
}

static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
{
	struct irq_desc *desc = irq_to_desc(irq);
	struct irq_cfg *cfg;
	unsigned int dest;
	cpumask_t tmp;

	cpus_and(tmp, mask, cpu_online_map);
	if (cpus_empty(tmp))
		return;

	cfg = desc->chip_data;
	if (assign_irq_vector(irq, cfg, mask))
		return;

	set_extra_move_desc(desc, mask);

	cpus_and(tmp, cfg->domain, mask);
	dest = cpu_mask_to_apicid(tmp);

	target_ht_irq(irq, dest, cfg->vector);
	desc->affinity = mask;
}

#endif

static struct irq_chip ht_irq_chip = {
	.name		= "PCI-HT",
	.mask		= mask_ht_irq,
	.unmask		= unmask_ht_irq,
	.ack		= ack_apic_edge,
#ifdef CONFIG_SMP
	.set_affinity	= set_ht_irq_affinity,
#endif
	.retrigger	= ioapic_retrigger_irq,
};

int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
{
	struct irq_cfg *cfg;
	int err;
	cpumask_t tmp;

	cfg = irq_cfg(irq);
	tmp = TARGET_CPUS;
	err = assign_irq_vector(irq, cfg, tmp);
	if (!err) {
		struct ht_irq_msg msg;
		unsigned dest;

		cpus_and(tmp, cfg->domain, tmp);
		dest = cpu_mask_to_apicid(tmp);

		msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);

		msg.address_lo =
			HT_IRQ_LOW_BASE |
			HT_IRQ_LOW_DEST_ID(dest) |
			HT_IRQ_LOW_VECTOR(cfg->vector) |
			((INT_DEST_MODE == 0) ?
				HT_IRQ_LOW_DM_PHYSICAL :
				HT_IRQ_LOW_DM_LOGICAL) |
			HT_IRQ_LOW_RQEOI_EDGE |
			((INT_DELIVERY_MODE != dest_LowestPrio) ?
				HT_IRQ_LOW_MT_FIXED :
				HT_IRQ_LOW_MT_ARBITRATED) |
			HT_IRQ_LOW_IRQ_MASKED;

		write_ht_irq_msg(irq, &msg);

		set_irq_chip_and_handler_name(irq, &ht_irq_chip,
					      handle_edge_irq, "edge");

		dev_printk(KERN_DEBUG, &dev->dev, "irq %d for HT\n", irq);
	}
	return err;
}
#endif /* CONFIG_HT_IRQ */

#ifdef CONFIG_X86_64
/*
 * Re-target the irq to the specified CPU and enable the specified MMR located
 * on the specified blade to allow the sending of MSIs to the specified CPU.
 */
int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
		       unsigned long mmr_offset)
{
	const cpumask_t *eligible_cpu = get_cpu_mask(cpu);
	struct irq_cfg *cfg;
	int mmr_pnode;
	unsigned long mmr_value;
	struct uv_IO_APIC_route_entry *entry;
	unsigned long flags;
	int err;

	cfg = irq_cfg(irq);

	err = assign_irq_vector(irq, cfg, *eligible_cpu);
	if (err != 0)
		return err;

	spin_lock_irqsave(&vector_lock, flags);
	set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
				      irq_name);
	spin_unlock_irqrestore(&vector_lock, flags);

	mmr_value = 0;
	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));

	entry->vector = cfg->vector;
	entry->delivery_mode = INT_DELIVERY_MODE;
	entry->dest_mode = INT_DEST_MODE;
	entry->polarity = 0;
	entry->trigger = 0;
	entry->mask = 0;
	entry->dest = cpu_mask_to_apicid(*eligible_cpu);

	mmr_pnode = uv_blade_to_pnode(mmr_blade);
	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);

	return irq;
}

/*
 * Disable the specified MMR located on the specified blade so that MSIs are
 * longer allowed to be sent.
 */
void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
{
	unsigned long mmr_value;
	struct uv_IO_APIC_route_entry *entry;
	int mmr_pnode;

	mmr_value = 0;
	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));

	entry->mask = 1;

	mmr_pnode = uv_blade_to_pnode(mmr_blade);
	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
}
#endif /* CONFIG_X86_64 */

int __init io_apic_get_redir_entries (int ioapic)
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_01.raw = io_apic_read(ioapic, 1);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return reg_01.bits.entries;
}

void __init probe_nr_irqs_gsi(void)
{
	int idx;
	int nr = 0;

	for (idx = 0; idx < nr_ioapics; idx++)
		nr += io_apic_get_redir_entries(idx) + 1;

	if (nr > nr_irqs_gsi)
		nr_irqs_gsi = nr;
}

/* --------------------------------------------------------------------------
                          ACPI-based IOAPIC Configuration
   -------------------------------------------------------------------------- */

#ifdef CONFIG_ACPI

#ifdef CONFIG_X86_32
int __init io_apic_get_unique_id(int ioapic, int apic_id)
{
	union IO_APIC_reg_00 reg_00;
	static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
	physid_mask_t tmp;
	unsigned long flags;
	int i = 0;

	/*
	 * The P4 platform supports up to 256 APIC IDs on two separate APIC
	 * buses (one for LAPICs, one for IOAPICs), where predecessors only
	 * supports up to 16 on one shared APIC bus.
	 *
	 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
	 *      advantage of new APIC bus architecture.
	 */

	if (physids_empty(apic_id_map))
		apic_id_map = ioapic_phys_id_map(phys_cpu_present_map);

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_00.raw = io_apic_read(ioapic, 0);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	if (apic_id >= get_physical_broadcast()) {
		printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
			"%d\n", ioapic, apic_id, reg_00.bits.ID);
		apic_id = reg_00.bits.ID;
	}

	/*
	 * Every APIC in a system must have a unique ID or we get lots of nice
	 * 'stuck on smp_invalidate_needed IPI wait' messages.
	 */
	if (check_apicid_used(apic_id_map, apic_id)) {

		for (i = 0; i < get_physical_broadcast(); i++) {
			if (!check_apicid_used(apic_id_map, i))
				break;
		}

		if (i == get_physical_broadcast())
			panic("Max apic_id exceeded!\n");

		printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
			"trying %d\n", ioapic, apic_id, i);

		apic_id = i;
	}

	tmp = apicid_to_cpu_present(apic_id);
	physids_or(apic_id_map, apic_id_map, tmp);

	if (reg_00.bits.ID != apic_id) {
		reg_00.bits.ID = apic_id;

		spin_lock_irqsave(&ioapic_lock, flags);
		io_apic_write(ioapic, 0, reg_00.raw);
		reg_00.raw = io_apic_read(ioapic, 0);
		spin_unlock_irqrestore(&ioapic_lock, flags);

		/* Sanity check */
		if (reg_00.bits.ID != apic_id) {
			printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
			return -1;
		}
	}

	apic_printk(APIC_VERBOSE, KERN_INFO
			"IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);

	return apic_id;
}

int __init io_apic_get_version(int ioapic)
{
	union IO_APIC_reg_01	reg_01;
	unsigned long flags;

	spin_lock_irqsave(&ioapic_lock, flags);
	reg_01.raw = io_apic_read(ioapic, 1);
	spin_unlock_irqrestore(&ioapic_lock, flags);

	return reg_01.bits.version;
}
#endif

int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
{
	struct irq_desc *desc;
	struct irq_cfg *cfg;
	int cpu = boot_cpu_id;

	if (!IO_APIC_IRQ(irq)) {
		apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
			ioapic);
		return -EINVAL;
	}

	desc = irq_to_desc_alloc_cpu(irq, cpu);
	if (!desc) {
		printk(KERN_INFO "can not get irq_desc %d\n", irq);
		return 0;
	}

	/*
	 * IRQs < 16 are already in the irq_2_pin[] map
	 */
	if (irq >= NR_IRQS_LEGACY) {
		cfg = desc->chip_data;
		add_pin_to_irq_cpu(cfg, cpu, ioapic, pin);
	}

	setup_IO_APIC_irq(ioapic, pin, irq, desc, triggering, polarity);

	return 0;
}


int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
{
	int i;

	if (skip_ioapic_setup)
		return -1;

	for (i = 0; i < mp_irq_entries; i++)
		if (mp_irqs[i].mp_irqtype == mp_INT &&
		    mp_irqs[i].mp_srcbusirq == bus_irq)
			break;
	if (i >= mp_irq_entries)
		return -1;

	*trigger = irq_trigger(i);
	*polarity = irq_polarity(i);
	return 0;
}

#endif /* CONFIG_ACPI */

/*
 * This function currently is only a helper for the i386 smp boot process where
 * we need to reprogram the ioredtbls to cater for the cpus which have come online
 * so mask in all cases should simply be TARGET_CPUS
 */
#ifdef CONFIG_SMP
void __init setup_ioapic_dest(void)
{
	int pin, ioapic, irq, irq_entry;
	struct irq_desc *desc;
	struct irq_cfg *cfg;
	cpumask_t mask;

	if (skip_ioapic_setup == 1)
		return;

	for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
		for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
			irq_entry = find_irq_entry(ioapic, pin, mp_INT);
			if (irq_entry == -1)
				continue;
			irq = pin_2_irq(irq_entry, ioapic, pin);

			/* setup_IO_APIC_irqs could fail to get vector for some device
			 * when you have too many devices, because at that time only boot
			 * cpu is online.
			 */
			desc = irq_to_desc(irq);
			cfg = desc->chip_data;
			if (!cfg->vector) {
				setup_IO_APIC_irq(ioapic, pin, irq, desc,
						  irq_trigger(irq_entry),
						  irq_polarity(irq_entry));
				continue;

			}

			/*
			 * Honour affinities which have been set in early boot
			 */
			if (desc->status &
			    (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
				mask = desc->affinity;
			else
				mask = TARGET_CPUS;

#ifdef CONFIG_INTR_REMAP
			if (intr_remapping_enabled)
				set_ir_ioapic_affinity_irq_desc(desc, mask);
			else
#endif
				set_ioapic_affinity_irq_desc(desc, mask);
		}

	}
}
#endif

#define IOAPIC_RESOURCE_NAME_SIZE 11

static struct resource *ioapic_resources;

static struct resource * __init ioapic_setup_resources(void)
{
	unsigned long n;
	struct resource *res;
	char *mem;
	int i;

	if (nr_ioapics <= 0)
		return NULL;

	n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
	n *= nr_ioapics;

	mem = alloc_bootmem(n);
	res = (void *)mem;

	if (mem != NULL) {
		mem += sizeof(struct resource) * nr_ioapics;

		for (i = 0; i < nr_ioapics; i++) {
			res[i].name = mem;
			res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
			sprintf(mem,  "IOAPIC %u", i);
			mem += IOAPIC_RESOURCE_NAME_SIZE;
		}
	}

	ioapic_resources = res;

	return res;
}

void __init ioapic_init_mappings(void)
{
	unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
	struct resource *ioapic_res;
	int i;

	ioapic_res = ioapic_setup_resources();
	for (i = 0; i < nr_ioapics; i++) {
		if (smp_found_config) {
			ioapic_phys = mp_ioapics[i].mp_apicaddr;
#ifdef CONFIG_X86_32
			if (!ioapic_phys) {
				printk(KERN_ERR
				       "WARNING: bogus zero IO-APIC "
				       "address found in MPTABLE, "
				       "disabling IO/APIC support!\n");
				smp_found_config = 0;
				skip_ioapic_setup = 1;
				goto fake_ioapic_page;
			}
#endif
		} else {
#ifdef CONFIG_X86_32
fake_ioapic_page:
#endif
			ioapic_phys = (unsigned long)
				alloc_bootmem_pages(PAGE_SIZE);
			ioapic_phys = __pa(ioapic_phys);
		}
		set_fixmap_nocache(idx, ioapic_phys);
		apic_printk(APIC_VERBOSE,
			    "mapped IOAPIC to %08lx (%08lx)\n",
			    __fix_to_virt(idx), ioapic_phys);
		idx++;

		if (ioapic_res != NULL) {
			ioapic_res->start = ioapic_phys;
			ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
			ioapic_res++;
		}
	}
}

static int __init ioapic_insert_resources(void)
{
	int i;
	struct resource *r = ioapic_resources;

	if (!r) {
		printk(KERN_ERR
		       "IO APIC resources could be not be allocated.\n");
		return -1;
	}

	for (i = 0; i < nr_ioapics; i++) {
		insert_resource(&iomem_resource, r);
		r++;
	}

	return 0;
}

/* Insert the IO APIC resources after PCI initialization has occured to handle
 * IO APICS that are mapped in on a BAR in PCI space. */
late_initcall(ioapic_insert_resources);