aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/pci/pcie/aspm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/pci/pcie/aspm.c')
-rw-r--r--drivers/pci/pcie/aspm.c787
1 files changed, 371 insertions, 416 deletions
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index b0367f168af4..3d27c97e0486 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -26,40 +26,36 @@
26#endif 26#endif
27#define MODULE_PARAM_PREFIX "pcie_aspm." 27#define MODULE_PARAM_PREFIX "pcie_aspm."
28 28
29struct endpoint_state { 29struct aspm_latency {
30 unsigned int l0s_acceptable_latency; 30 u32 l0s; /* L0s latency (nsec) */
31 unsigned int l1_acceptable_latency; 31 u32 l1; /* L1 latency (nsec) */
32}; 32};
33 33
34struct pcie_link_state { 34struct pcie_link_state {
35 struct list_head sibiling; 35 struct pci_dev *pdev; /* Upstream component of the Link */
36 struct pci_dev *pdev; 36 struct pcie_link_state *root; /* pointer to the root port link */
37 bool downstream_has_switch; 37 struct pcie_link_state *parent; /* pointer to the parent Link state */
38 38 struct list_head sibling; /* node in link_list */
39 struct pcie_link_state *parent; 39 struct list_head children; /* list of child link states */
40 struct list_head children; 40 struct list_head link; /* node in parent's children list */
41 struct list_head link;
42 41
43 /* ASPM state */ 42 /* ASPM state */
44 unsigned int support_state; 43 u32 aspm_support:2; /* Supported ASPM state */
45 unsigned int enabled_state; 44 u32 aspm_enabled:2; /* Enabled ASPM state */
46 unsigned int bios_aspm_state; 45 u32 aspm_default:2; /* Default ASPM state by BIOS */
47 /* upstream component */ 46
48 unsigned int l0s_upper_latency; 47 /* Clock PM state */
49 unsigned int l1_upper_latency; 48 u32 clkpm_capable:1; /* Clock PM capable? */
50 /* downstream component */ 49 u32 clkpm_enabled:1; /* Current Clock PM state */
51 unsigned int l0s_down_latency; 50 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
52 unsigned int l1_down_latency;
53 /* Clock PM state*/
54 unsigned int clk_pm_capable;
55 unsigned int clk_pm_enabled;
56 unsigned int bios_clk_state;
57 51
52 /* Latencies */
53 struct aspm_latency latency; /* Exit latency */
58 /* 54 /*
59 * A pcie downstream port only has one slot under it, so at most there 55 * Endpoint acceptable latencies. A pcie downstream port only
60 * are 8 functions 56 * has one slot under it, so at most there are 8 functions.
61 */ 57 */
62 struct endpoint_state endpoints[8]; 58 struct aspm_latency acceptable[8];
63}; 59};
64 60
65static int aspm_disabled, aspm_force; 61static int aspm_disabled, aspm_force;
@@ -78,27 +74,23 @@ static const char *policy_str[] = {
78 74
79#define LINK_RETRAIN_TIMEOUT HZ 75#define LINK_RETRAIN_TIMEOUT HZ
80 76
81static int policy_to_aspm_state(struct pci_dev *pdev) 77static int policy_to_aspm_state(struct pcie_link_state *link)
82{ 78{
83 struct pcie_link_state *link_state = pdev->link_state;
84
85 switch (aspm_policy) { 79 switch (aspm_policy) {
86 case POLICY_PERFORMANCE: 80 case POLICY_PERFORMANCE:
87 /* Disable ASPM and Clock PM */ 81 /* Disable ASPM and Clock PM */
88 return 0; 82 return 0;
89 case POLICY_POWERSAVE: 83 case POLICY_POWERSAVE:
90 /* Enable ASPM L0s/L1 */ 84 /* Enable ASPM L0s/L1 */
91 return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1; 85 return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
92 case POLICY_DEFAULT: 86 case POLICY_DEFAULT:
93 return link_state->bios_aspm_state; 87 return link->aspm_default;
94 } 88 }
95 return 0; 89 return 0;
96} 90}
97 91
98static int policy_to_clkpm_state(struct pci_dev *pdev) 92static int policy_to_clkpm_state(struct pcie_link_state *link)
99{ 93{
100 struct pcie_link_state *link_state = pdev->link_state;
101
102 switch (aspm_policy) { 94 switch (aspm_policy) {
103 case POLICY_PERFORMANCE: 95 case POLICY_PERFORMANCE:
104 /* Disable ASPM and Clock PM */ 96 /* Disable ASPM and Clock PM */
@@ -107,73 +99,78 @@ static int policy_to_clkpm_state(struct pci_dev *pdev)
107 /* Disable Clock PM */ 99 /* Disable Clock PM */
108 return 1; 100 return 1;
109 case POLICY_DEFAULT: 101 case POLICY_DEFAULT:
110 return link_state->bios_clk_state; 102 return link->clkpm_default;
111 } 103 }
112 return 0; 104 return 0;
113} 105}
114 106
115static void pcie_set_clock_pm(struct pci_dev *pdev, int enable) 107static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
116{ 108{
117 struct pci_dev *child_dev;
118 int pos; 109 int pos;
119 u16 reg16; 110 u16 reg16;
120 struct pcie_link_state *link_state = pdev->link_state; 111 struct pci_dev *child;
112 struct pci_bus *linkbus = link->pdev->subordinate;
121 113
122 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 114 list_for_each_entry(child, &linkbus->devices, bus_list) {
123 pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); 115 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
124 if (!pos) 116 if (!pos)
125 return; 117 return;
126 pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, &reg16); 118 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
127 if (enable) 119 if (enable)
128 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN; 120 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
129 else 121 else
130 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN; 122 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
131 pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16); 123 pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
132 } 124 }
133 link_state->clk_pm_enabled = !!enable; 125 link->clkpm_enabled = !!enable;
134} 126}
135 127
136static void pcie_check_clock_pm(struct pci_dev *pdev, int blacklist) 128static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
137{ 129{
138 int pos; 130 /* Don't enable Clock PM if the link is not Clock PM capable */
131 if (!link->clkpm_capable && enable)
132 return;
133 /* Need nothing if the specified equals to current state */
134 if (link->clkpm_enabled == enable)
135 return;
136 pcie_set_clkpm_nocheck(link, enable);
137}
138
139static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
140{
141 int pos, capable = 1, enabled = 1;
139 u32 reg32; 142 u32 reg32;
140 u16 reg16; 143 u16 reg16;
141 int capable = 1, enabled = 1; 144 struct pci_dev *child;
142 struct pci_dev *child_dev; 145 struct pci_bus *linkbus = link->pdev->subordinate;
143 struct pcie_link_state *link_state = pdev->link_state;
144 146
145 /* All functions should have the same cap and state, take the worst */ 147 /* All functions should have the same cap and state, take the worst */
146 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 148 list_for_each_entry(child, &linkbus->devices, bus_list) {
147 pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); 149 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
148 if (!pos) 150 if (!pos)
149 return; 151 return;
150 pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, &reg32); 152 pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, &reg32);
151 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 153 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
152 capable = 0; 154 capable = 0;
153 enabled = 0; 155 enabled = 0;
154 break; 156 break;
155 } 157 }
156 pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, &reg16); 158 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
157 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 159 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
158 enabled = 0; 160 enabled = 0;
159 } 161 }
160 link_state->clk_pm_enabled = enabled; 162 link->clkpm_enabled = enabled;
161 link_state->bios_clk_state = enabled; 163 link->clkpm_default = enabled;
162 if (!blacklist) { 164 link->clkpm_capable = (blacklist) ? 0 : capable;
163 link_state->clk_pm_capable = capable;
164 pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
165 } else {
166 link_state->clk_pm_capable = 0;
167 pcie_set_clock_pm(pdev, 0);
168 }
169} 165}
170 166
171static bool pcie_aspm_downstream_has_switch(struct pci_dev *pdev) 167static bool pcie_aspm_downstream_has_switch(struct pcie_link_state *link)
172{ 168{
173 struct pci_dev *child_dev; 169 struct pci_dev *child;
170 struct pci_bus *linkbus = link->pdev->subordinate;
174 171
175 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 172 list_for_each_entry(child, &linkbus->devices, bus_list) {
176 if (child_dev->pcie_type == PCI_EXP_TYPE_UPSTREAM) 173 if (child->pcie_type == PCI_EXP_TYPE_UPSTREAM)
177 return true; 174 return true;
178 } 175 }
179 return false; 176 return false;
@@ -184,289 +181,263 @@ static bool pcie_aspm_downstream_has_switch(struct pci_dev *pdev)
184 * could use common clock. If they are, configure them to use the 181 * could use common clock. If they are, configure them to use the
185 * common clock. That will reduce the ASPM state exit latency. 182 * common clock. That will reduce the ASPM state exit latency.
186 */ 183 */
187static void pcie_aspm_configure_common_clock(struct pci_dev *pdev) 184static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
188{ 185{
189 int pos, child_pos, i = 0; 186 int ppos, cpos, same_clock = 1;
190 u16 reg16 = 0; 187 u16 reg16, parent_reg, child_reg[8];
191 struct pci_dev *child_dev;
192 int same_clock = 1;
193 unsigned long start_jiffies; 188 unsigned long start_jiffies;
194 u16 child_regs[8], parent_reg; 189 struct pci_dev *child, *parent = link->pdev;
190 struct pci_bus *linkbus = parent->subordinate;
195 /* 191 /*
196 * all functions of a slot should have the same Slot Clock 192 * All functions of a slot should have the same Slot Clock
197 * Configuration, so just check one function 193 * Configuration, so just check one function
198 * */ 194 */
199 child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev, 195 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
200 bus_list); 196 BUG_ON(!child->is_pcie);
201 BUG_ON(!child_dev->is_pcie);
202 197
203 /* Check downstream component if bit Slot Clock Configuration is 1 */ 198 /* Check downstream component if bit Slot Clock Configuration is 1 */
204 child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); 199 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
205 pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, &reg16); 200 pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, &reg16);
206 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 201 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
207 same_clock = 0; 202 same_clock = 0;
208 203
209 /* Check upstream component if bit Slot Clock Configuration is 1 */ 204 /* Check upstream component if bit Slot Clock Configuration is 1 */
210 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); 205 ppos = pci_find_capability(parent, PCI_CAP_ID_EXP);
211 pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, &reg16); 206 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
212 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 207 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
213 same_clock = 0; 208 same_clock = 0;
214 209
215 /* Configure downstream component, all functions */ 210 /* Configure downstream component, all functions */
216 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 211 list_for_each_entry(child, &linkbus->devices, bus_list) {
217 child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); 212 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
218 pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL, 213 pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, &reg16);
219 &reg16); 214 child_reg[PCI_FUNC(child->devfn)] = reg16;
220 child_regs[i] = reg16;
221 if (same_clock) 215 if (same_clock)
222 reg16 |= PCI_EXP_LNKCTL_CCC; 216 reg16 |= PCI_EXP_LNKCTL_CCC;
223 else 217 else
224 reg16 &= ~PCI_EXP_LNKCTL_CCC; 218 reg16 &= ~PCI_EXP_LNKCTL_CCC;
225 pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL, 219 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
226 reg16);
227 i++;
228 } 220 }
229 221
230 /* Configure upstream component */ 222 /* Configure upstream component */
231 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16); 223 pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, &reg16);
232 parent_reg = reg16; 224 parent_reg = reg16;
233 if (same_clock) 225 if (same_clock)
234 reg16 |= PCI_EXP_LNKCTL_CCC; 226 reg16 |= PCI_EXP_LNKCTL_CCC;
235 else 227 else
236 reg16 &= ~PCI_EXP_LNKCTL_CCC; 228 reg16 &= ~PCI_EXP_LNKCTL_CCC;
237 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); 229 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
238 230
239 /* retrain link */ 231 /* Retrain link */
240 reg16 |= PCI_EXP_LNKCTL_RL; 232 reg16 |= PCI_EXP_LNKCTL_RL;
241 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); 233 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
242 234
243 /* Wait for link training end */ 235 /* Wait for link training end. Break out after waiting for timeout */
244 /* break out after waiting for timeout */
245 start_jiffies = jiffies; 236 start_jiffies = jiffies;
246 for (;;) { 237 for (;;) {
247 pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, &reg16); 238 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
248 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 239 if (!(reg16 & PCI_EXP_LNKSTA_LT))
249 break; 240 break;
250 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 241 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
251 break; 242 break;
252 msleep(1); 243 msleep(1);
253 } 244 }
254 /* training failed -> recover */ 245 if (!(reg16 & PCI_EXP_LNKSTA_LT))
255 if (reg16 & PCI_EXP_LNKSTA_LT) { 246 return;
256 dev_printk (KERN_ERR, &pdev->dev, "ASPM: Could not configure" 247
257 " common clock\n"); 248 /* Training failed. Restore common clock configurations */
258 i = 0; 249 dev_printk(KERN_ERR, &parent->dev,
259 list_for_each_entry(child_dev, &pdev->subordinate->devices, 250 "ASPM: Could not configure common clock\n");
260 bus_list) { 251 list_for_each_entry(child, &linkbus->devices, bus_list) {
261 child_pos = pci_find_capability(child_dev, 252 cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
262 PCI_CAP_ID_EXP); 253 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
263 pci_write_config_word(child_dev, 254 child_reg[PCI_FUNC(child->devfn)]);
264 child_pos + PCI_EXP_LNKCTL,
265 child_regs[i]);
266 i++;
267 }
268 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, parent_reg);
269 } 255 }
256 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
270} 257}
271 258
272/* 259/* Convert L0s latency encoding to ns */
273 * calc_L0S_latency: Convert L0s latency encoding to ns 260static u32 calc_l0s_latency(u32 encoding)
274 */
275static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
276{ 261{
277 unsigned int ns = 64; 262 if (encoding == 0x7)
263 return (5 * 1000); /* > 4us */
264 return (64 << encoding);
265}
278 266
279 if (latency_encoding == 0x7) { 267/* Convert L0s acceptable latency encoding to ns */
280 if (ac) 268static u32 calc_l0s_acceptable(u32 encoding)
281 ns = -1U; 269{
282 else 270 if (encoding == 0x7)
283 ns = 5*1000; /* > 4us */ 271 return -1U;
284 } else 272 return (64 << encoding);
285 ns *= (1 << latency_encoding);
286 return ns;
287} 273}
288 274
289/* 275/* Convert L1 latency encoding to ns */
290 * calc_L1_latency: Convert L1 latency encoding to ns 276static u32 calc_l1_latency(u32 encoding)
291 */
292static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
293{ 277{
294 unsigned int ns = 1000; 278 if (encoding == 0x7)
279 return (65 * 1000); /* > 64us */
280 return (1000 << encoding);
281}
295 282
296 if (latency_encoding == 0x7) { 283/* Convert L1 acceptable latency encoding to ns */
297 if (ac) 284static u32 calc_l1_acceptable(u32 encoding)
298 ns = -1U; 285{
299 else 286 if (encoding == 0x7)
300 ns = 65*1000; /* > 64us */ 287 return -1U;
301 } else 288 return (1000 << encoding);
302 ns *= (1 << latency_encoding);
303 return ns;
304} 289}
305 290
306static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state, 291static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
307 unsigned int *l0s, unsigned int *l1, unsigned int *enabled) 292 u32 *l0s, u32 *l1, u32 *enabled)
308{ 293{
309 int pos; 294 int pos;
310 u16 reg16; 295 u16 reg16;
311 u32 reg32; 296 u32 reg32, encoding;
312 unsigned int latency;
313 297
298 *l0s = *l1 = *enabled = 0;
314 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); 299 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
315 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, &reg32); 300 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, &reg32);
316 *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 301 *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
317 if (*state != PCIE_LINK_STATE_L0S && 302 if (*state != PCIE_LINK_STATE_L0S &&
318 *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S)) 303 *state != (PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_L0S))
319 *state = 0; 304 *state = 0;
320 if (*state == 0) 305 if (*state == 0)
321 return; 306 return;
322 307
323 latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 308 encoding = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
324 *l0s = calc_L0S_latency(latency, 0); 309 *l0s = calc_l0s_latency(encoding);
325 if (*state & PCIE_LINK_STATE_L1) { 310 if (*state & PCIE_LINK_STATE_L1) {
326 latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 311 encoding = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
327 *l1 = calc_L1_latency(latency, 0); 312 *l1 = calc_l1_latency(encoding);
328 } 313 }
329 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16); 314 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
330 *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1); 315 *enabled = reg16 & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
331} 316}
332 317
333static void pcie_aspm_cap_init(struct pci_dev *pdev) 318static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
334{ 319{
335 struct pci_dev *child_dev; 320 u32 support, l0s, l1, enabled;
336 u32 state, tmp; 321 struct pci_dev *child, *parent = link->pdev;
337 struct pcie_link_state *link_state = pdev->link_state; 322 struct pci_bus *linkbus = parent->subordinate;
323
324 if (blacklist) {
325 /* Set support state to 0, so we will disable ASPM later */
326 link->aspm_support = 0;
327 link->aspm_default = 0;
328 link->aspm_enabled = PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
329 return;
330 }
331
332 /* Configure common clock before checking latencies */
333 pcie_aspm_configure_common_clock(link);
338 334
339 /* upstream component states */ 335 /* upstream component states */
340 pcie_aspm_get_cap_device(pdev, &link_state->support_state, 336 pcie_aspm_get_cap_device(parent, &support, &l0s, &l1, &enabled);
341 &link_state->l0s_upper_latency, 337 link->aspm_support = support;
342 &link_state->l1_upper_latency, 338 link->latency.l0s = l0s;
343 &link_state->enabled_state); 339 link->latency.l1 = l1;
340 link->aspm_enabled = enabled;
341
344 /* downstream component states, all functions have the same setting */ 342 /* downstream component states, all functions have the same setting */
345 child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev, 343 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
346 bus_list); 344 pcie_aspm_get_cap_device(child, &support, &l0s, &l1, &enabled);
347 pcie_aspm_get_cap_device(child_dev, &state, 345 link->aspm_support &= support;
348 &link_state->l0s_down_latency, 346 link->latency.l0s = max_t(u32, link->latency.l0s, l0s);
349 &link_state->l1_down_latency, 347 link->latency.l1 = max_t(u32, link->latency.l1, l1);
350 &tmp); 348
351 link_state->support_state &= state; 349 if (!link->aspm_support)
352 if (!link_state->support_state)
353 return; 350 return;
354 link_state->enabled_state &= link_state->support_state; 351
355 link_state->bios_aspm_state = link_state->enabled_state; 352 link->aspm_enabled &= link->aspm_support;
353 link->aspm_default = link->aspm_enabled;
356 354
357 /* ENDPOINT states*/ 355 /* ENDPOINT states*/
358 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 356 list_for_each_entry(child, &linkbus->devices, bus_list) {
359 int pos; 357 int pos;
360 u32 reg32; 358 u32 reg32, encoding;
361 unsigned int latency; 359 struct aspm_latency *acceptable =
362 struct endpoint_state *ep_state = 360 &link->acceptable[PCI_FUNC(child->devfn)];
363 &link_state->endpoints[PCI_FUNC(child_dev->devfn)];
364 361
365 if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT && 362 if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
366 child_dev->pcie_type != PCI_EXP_TYPE_LEG_END) 363 child->pcie_type != PCI_EXP_TYPE_LEG_END)
367 continue; 364 continue;
368 365
369 pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); 366 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
370 pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, &reg32); 367 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
371 latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 368 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
372 latency = calc_L0S_latency(latency, 1); 369 acceptable->l0s = calc_l0s_acceptable(encoding);
373 ep_state->l0s_acceptable_latency = latency; 370 if (link->aspm_support & PCIE_LINK_STATE_L1) {
374 if (link_state->support_state & PCIE_LINK_STATE_L1) { 371 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
375 latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 372 acceptable->l1 = calc_l1_acceptable(encoding);
376 latency = calc_L1_latency(latency, 1);
377 ep_state->l1_acceptable_latency = latency;
378 } 373 }
379 } 374 }
380} 375}
381 376
382static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev, 377/**
383 unsigned int state) 378 * __pcie_aspm_check_state_one - check latency for endpoint device.
384{ 379 * @endpoint: pointer to the struct pci_dev of endpoint device
385 struct pci_dev *parent_dev, *tmp_dev; 380 *
386 unsigned int latency, l1_latency = 0; 381 * TBD: The latency from the endpoint to root complex vary per switch's
387 struct pcie_link_state *link_state; 382 * upstream link state above the device. Here we just do a simple check
388 struct endpoint_state *ep_state; 383 * which assumes all links above the device can be in L1 state, that
389 384 * is we just consider the worst case. If switch's upstream link can't
390 parent_dev = pdev->bus->self; 385 * be put into L0S/L1, then our check is too strictly.
391 link_state = parent_dev->link_state; 386 */
392 state &= link_state->support_state; 387static u32 __pcie_aspm_check_state_one(struct pci_dev *endpoint, u32 state)
393 if (state == 0) 388{
394 return 0; 389 u32 l1_switch_latency = 0;
395 ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)]; 390 struct aspm_latency *acceptable;
396 391 struct pcie_link_state *link;
397 /* 392
398 * Check latency for endpoint device. 393 link = endpoint->bus->self->link_state;
399 * TBD: The latency from the endpoint to root complex vary per 394 state &= link->aspm_support;
400 * switch's upstream link state above the device. Here we just do a 395 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
401 * simple check which assumes all links above the device can be in L1 396
402 * state, that is we just consider the worst case. If switch's upstream 397 while (link && state) {
403 * link can't be put into L0S/L1, then our check is too strictly. 398 if ((state & PCIE_LINK_STATE_L0S) &&
404 */ 399 (link->latency.l0s > acceptable->l0s))
405 tmp_dev = pdev; 400 state &= ~PCIE_LINK_STATE_L0S;
406 while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) { 401 if ((state & PCIE_LINK_STATE_L1) &&
407 parent_dev = tmp_dev->bus->self; 402 (link->latency.l1 + l1_switch_latency > acceptable->l1))
408 link_state = parent_dev->link_state; 403 state &= ~PCIE_LINK_STATE_L1;
409 if (state & PCIE_LINK_STATE_L0S) { 404 link = link->parent;
410 latency = max_t(unsigned int, 405 /*
411 link_state->l0s_upper_latency, 406 * Every switch on the path to root complex need 1
412 link_state->l0s_down_latency); 407 * more microsecond for L1. Spec doesn't mention L0s.
413 if (latency > ep_state->l0s_acceptable_latency) 408 */
414 state &= ~PCIE_LINK_STATE_L0S; 409 l1_switch_latency += 1000;
415 }
416 if (state & PCIE_LINK_STATE_L1) {
417 latency = max_t(unsigned int,
418 link_state->l1_upper_latency,
419 link_state->l1_down_latency);
420 if (latency + l1_latency >
421 ep_state->l1_acceptable_latency)
422 state &= ~PCIE_LINK_STATE_L1;
423 }
424 if (!parent_dev->bus->self) /* parent_dev is a root port */
425 break;
426 else {
427 /*
428 * parent_dev is the downstream port of a switch, make
429 * tmp_dev the upstream port of the switch
430 */
431 tmp_dev = parent_dev->bus->self;
432 /*
433 * every switch on the path to root complex need 1 more
434 * microsecond for L1. Spec doesn't mention L0S.
435 */
436 if (state & PCIE_LINK_STATE_L1)
437 l1_latency += 1000;
438 }
439 } 410 }
440 return state; 411 return state;
441} 412}
442 413
443static unsigned int pcie_aspm_check_state(struct pci_dev *pdev, 414static u32 pcie_aspm_check_state(struct pcie_link_state *link, u32 state)
444 unsigned int state)
445{ 415{
446 struct pci_dev *child_dev; 416 pci_power_t power_state;
417 struct pci_dev *child;
418 struct pci_bus *linkbus = link->pdev->subordinate;
447 419
448 /* If no child, ignore the link */ 420 /* If no child, ignore the link */
449 if (list_empty(&pdev->subordinate->devices)) 421 if (list_empty(&linkbus->devices))
450 return state; 422 return state;
451 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 423
452 if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) { 424 list_for_each_entry(child, &linkbus->devices, bus_list) {
453 /* 425 /*
454 * If downstream component of a link is pci bridge, we 426 * If downstream component of a link is pci bridge, we
455 * disable ASPM for now for the link 427 * disable ASPM for now for the link
456 * */ 428 */
457 state = 0; 429 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
458 break; 430 return 0;
459 } 431
460 if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT && 432 if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
461 child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)) 433 child->pcie_type != PCI_EXP_TYPE_LEG_END))
462 continue; 434 continue;
463 /* Device not in D0 doesn't need check latency */ 435 /* Device not in D0 doesn't need check latency */
464 if (child_dev->current_state == PCI_D1 || 436 power_state = child->current_state;
465 child_dev->current_state == PCI_D2 || 437 if (power_state == PCI_D1 || power_state == PCI_D2 ||
466 child_dev->current_state == PCI_D3hot || 438 power_state == PCI_D3hot || power_state == PCI_D3cold)
467 child_dev->current_state == PCI_D3cold)
468 continue; 439 continue;
469 state = __pcie_aspm_check_state_one(child_dev, state); 440 state = __pcie_aspm_check_state_one(child, state);
470 } 441 }
471 return state; 442 return state;
472} 443}
@@ -482,90 +453,71 @@ static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
482 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); 453 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
483} 454}
484 455
485static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state) 456static void __pcie_aspm_config_link(struct pcie_link_state *link, u32 state)
486{ 457{
487 struct pci_dev *child_dev; 458 struct pci_dev *child, *parent = link->pdev;
488 int valid = 1; 459 struct pci_bus *linkbus = parent->subordinate;
489 struct pcie_link_state *link_state = pdev->link_state;
490 460
491 /* If no child, disable the link */ 461 /* If no child, disable the link */
492 if (list_empty(&pdev->subordinate->devices)) 462 if (list_empty(&linkbus->devices))
493 state = 0; 463 state = 0;
494 /* 464 /*
495 * if the downstream component has pci bridge function, don't do ASPM 465 * If the downstream component has pci bridge function, don't
496 * now 466 * do ASPM now.
497 */ 467 */
498 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 468 list_for_each_entry(child, &linkbus->devices, bus_list) {
499 if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) { 469 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
500 valid = 0; 470 return;
501 break;
502 }
503 } 471 }
504 if (!valid)
505 return;
506
507 /* 472 /*
508 * spec 2.0 suggests all functions should be configured the same 473 * Spec 2.0 suggests all functions should be configured the
509 * setting for ASPM. Enabling ASPM L1 should be done in upstream 474 * same setting for ASPM. Enabling ASPM L1 should be done in
510 * component first and then downstream, and vice versa for disabling 475 * upstream component first and then downstream, and vice
511 * ASPM L1. Spec doesn't mention L0S. 476 * versa for disabling ASPM L1. Spec doesn't mention L0S.
512 */ 477 */
513 if (state & PCIE_LINK_STATE_L1) 478 if (state & PCIE_LINK_STATE_L1)
514 __pcie_aspm_config_one_dev(pdev, state); 479 __pcie_aspm_config_one_dev(parent, state);
515 480
516 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) 481 list_for_each_entry(child, &linkbus->devices, bus_list)
517 __pcie_aspm_config_one_dev(child_dev, state); 482 __pcie_aspm_config_one_dev(child, state);
518 483
519 if (!(state & PCIE_LINK_STATE_L1)) 484 if (!(state & PCIE_LINK_STATE_L1))
520 __pcie_aspm_config_one_dev(pdev, state); 485 __pcie_aspm_config_one_dev(parent, state);
521 486
522 link_state->enabled_state = state; 487 link->aspm_enabled = state;
523} 488}
524 489
525static struct pcie_link_state *get_root_port_link(struct pcie_link_state *link) 490/* Check the whole hierarchy, and configure each link in the hierarchy */
491static void __pcie_aspm_configure_link_state(struct pcie_link_state *link,
492 u32 state)
526{ 493{
527 struct pcie_link_state *root_port_link = link; 494 struct pcie_link_state *leaf, *root = link->root;
528 while (root_port_link->parent)
529 root_port_link = root_port_link->parent;
530 return root_port_link;
531}
532 495
533/* check the whole hierarchy, and configure each link in the hierarchy */ 496 state &= (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
534static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
535 unsigned int state)
536{
537 struct pcie_link_state *link_state = pdev->link_state;
538 struct pcie_link_state *root_port_link = get_root_port_link(link_state);
539 struct pcie_link_state *leaf;
540 497
541 state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1; 498 /* Check all links who have specific root port link */
542 499 list_for_each_entry(leaf, &link_list, sibling) {
543 /* check all links who have specific root port link */ 500 if (!list_empty(&leaf->children) || (leaf->root != root))
544 list_for_each_entry(leaf, &link_list, sibiling) {
545 if (!list_empty(&leaf->children) ||
546 get_root_port_link(leaf) != root_port_link)
547 continue; 501 continue;
548 state = pcie_aspm_check_state(leaf->pdev, state); 502 state = pcie_aspm_check_state(leaf, state);
549 } 503 }
550 /* check root port link too in case it hasn't children */ 504 /* Check root port link too in case it hasn't children */
551 state = pcie_aspm_check_state(root_port_link->pdev, state); 505 state = pcie_aspm_check_state(root, state);
552 506 if (link->aspm_enabled == state)
553 if (link_state->enabled_state == state)
554 return; 507 return;
555
556 /* 508 /*
557 * we must change the hierarchy. See comments in 509 * We must change the hierarchy. See comments in
558 * __pcie_aspm_config_link for the order 510 * __pcie_aspm_config_link for the order
559 **/ 511 **/
560 if (state & PCIE_LINK_STATE_L1) { 512 if (state & PCIE_LINK_STATE_L1) {
561 list_for_each_entry(leaf, &link_list, sibiling) { 513 list_for_each_entry(leaf, &link_list, sibling) {
562 if (get_root_port_link(leaf) == root_port_link) 514 if (leaf->root == root)
563 __pcie_aspm_config_link(leaf->pdev, state); 515 __pcie_aspm_config_link(leaf, state);
564 } 516 }
565 } else { 517 } else {
566 list_for_each_entry_reverse(leaf, &link_list, sibiling) { 518 list_for_each_entry_reverse(leaf, &link_list, sibling) {
567 if (get_root_port_link(leaf) == root_port_link) 519 if (leaf->root == root)
568 __pcie_aspm_config_link(leaf->pdev, state); 520 __pcie_aspm_config_link(leaf, state);
569 } 521 }
570 } 522 }
571} 523}
@@ -574,45 +526,42 @@ static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
574 * pcie_aspm_configure_link_state: enable/disable PCI express link state 526 * pcie_aspm_configure_link_state: enable/disable PCI express link state
575 * @pdev: the root port or switch downstream port 527 * @pdev: the root port or switch downstream port
576 */ 528 */
577static void pcie_aspm_configure_link_state(struct pci_dev *pdev, 529static void pcie_aspm_configure_link_state(struct pcie_link_state *link,
578 unsigned int state) 530 u32 state)
579{ 531{
580 down_read(&pci_bus_sem); 532 down_read(&pci_bus_sem);
581 mutex_lock(&aspm_lock); 533 mutex_lock(&aspm_lock);
582 __pcie_aspm_configure_link_state(pdev, state); 534 __pcie_aspm_configure_link_state(link, state);
583 mutex_unlock(&aspm_lock); 535 mutex_unlock(&aspm_lock);
584 up_read(&pci_bus_sem); 536 up_read(&pci_bus_sem);
585} 537}
586 538
587static void free_link_state(struct pci_dev *pdev) 539static void free_link_state(struct pcie_link_state *link)
588{ 540{
589 kfree(pdev->link_state); 541 link->pdev->link_state = NULL;
590 pdev->link_state = NULL; 542 kfree(link);
591} 543}
592 544
593static int pcie_aspm_sanity_check(struct pci_dev *pdev) 545static int pcie_aspm_sanity_check(struct pci_dev *pdev)
594{ 546{
595 struct pci_dev *child_dev; 547 struct pci_dev *child;
596 int child_pos; 548 int pos;
597 u32 reg32; 549 u32 reg32;
598
599 /* 550 /*
600 * Some functions in a slot might not all be PCIE functions, very 551 * Some functions in a slot might not all be PCIE functions,
601 * strange. Disable ASPM for the whole slot 552 * very strange. Disable ASPM for the whole slot
602 */ 553 */
603 list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) { 554 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
604 child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP); 555 pos = pci_find_capability(child, PCI_CAP_ID_EXP);
605 if (!child_pos) 556 if (!pos)
606 return -EINVAL; 557 return -EINVAL;
607
608 /* 558 /*
609 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 559 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
610 * RBER bit to determine if a function is 1.1 version device 560 * RBER bit to determine if a function is 1.1 version device
611 */ 561 */
612 pci_read_config_dword(child_dev, child_pos + PCI_EXP_DEVCAP, 562 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
613 &reg32);
614 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 563 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
615 dev_printk(KERN_INFO, &child_dev->dev, "disabling ASPM" 564 dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
616 " on pre-1.1 PCIe device. You can enable it" 565 " on pre-1.1 PCIe device. You can enable it"
617 " with 'pcie_aspm=force'\n"); 566 " with 'pcie_aspm=force'\n");
618 return -EINVAL; 567 return -EINVAL;
@@ -621,6 +570,47 @@ static int pcie_aspm_sanity_check(struct pci_dev *pdev)
621 return 0; 570 return 0;
622} 571}
623 572
573static struct pcie_link_state *pcie_aspm_setup_link_state(struct pci_dev *pdev)
574{
575 struct pcie_link_state *link;
576 int blacklist = !!pcie_aspm_sanity_check(pdev);
577
578 link = kzalloc(sizeof(*link), GFP_KERNEL);
579 if (!link)
580 return NULL;
581 INIT_LIST_HEAD(&link->sibling);
582 INIT_LIST_HEAD(&link->children);
583 INIT_LIST_HEAD(&link->link);
584 link->pdev = pdev;
585 if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
586 struct pcie_link_state *parent;
587 parent = pdev->bus->parent->self->link_state;
588 if (!parent) {
589 kfree(link);
590 return NULL;
591 }
592 link->parent = parent;
593 list_add(&link->link, &parent->children);
594 }
595 /* Setup a pointer to the root port link */
596 if (!link->parent)
597 link->root = link;
598 else
599 link->root = link->parent->root;
600
601 list_add(&link->sibling, &link_list);
602
603 pdev->link_state = link;
604
605 /* Check ASPM capability */
606 pcie_aspm_cap_init(link, blacklist);
607
608 /* Check Clock PM capability */
609 pcie_clkpm_cap_init(link, blacklist);
610
611 return link;
612}
613
624/* 614/*
625 * pcie_aspm_init_link_state: Initiate PCI express link state. 615 * pcie_aspm_init_link_state: Initiate PCI express link state.
626 * It is called after the pcie and its children devices are scaned. 616 * It is called after the pcie and its children devices are scaned.
@@ -628,75 +618,47 @@ static int pcie_aspm_sanity_check(struct pci_dev *pdev)
628 */ 618 */
629void pcie_aspm_init_link_state(struct pci_dev *pdev) 619void pcie_aspm_init_link_state(struct pci_dev *pdev)
630{ 620{
631 unsigned int state; 621 u32 state;
632 struct pcie_link_state *link_state; 622 struct pcie_link_state *link;
633 int error = 0;
634 int blacklist;
635 623
636 if (aspm_disabled || !pdev->is_pcie || pdev->link_state) 624 if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
637 return; 625 return;
638 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && 626 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
639 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) 627 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
628 return;
629
630 /* VIA has a strange chipset, root port is under a bridge */
631 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
632 pdev->bus->self)
640 return; 633 return;
634
641 down_read(&pci_bus_sem); 635 down_read(&pci_bus_sem);
642 if (list_empty(&pdev->subordinate->devices)) 636 if (list_empty(&pdev->subordinate->devices))
643 goto out; 637 goto out;
644 638
645 blacklist = !!pcie_aspm_sanity_check(pdev);
646
647 mutex_lock(&aspm_lock); 639 mutex_lock(&aspm_lock);
648 640 link = pcie_aspm_setup_link_state(pdev);
649 link_state = kzalloc(sizeof(*link_state), GFP_KERNEL); 641 if (!link)
650 if (!link_state) 642 goto unlock;
651 goto unlock_out; 643 /*
652 644 * Setup initial ASPM state
653 link_state->downstream_has_switch = pcie_aspm_downstream_has_switch(pdev); 645 *
654 INIT_LIST_HEAD(&link_state->children); 646 * If link has switch, delay the link config. The leaf link
655 INIT_LIST_HEAD(&link_state->link); 647 * initialization will config the whole hierarchy. But we must
656 if (pdev->bus->self) {/* this is a switch */ 648 * make sure BIOS doesn't set unsupported link state.
657 struct pcie_link_state *parent_link_state; 649 */
658 650 if (pcie_aspm_downstream_has_switch(link)) {
659 parent_link_state = pdev->bus->parent->self->link_state; 651 state = pcie_aspm_check_state(link, link->aspm_default);
660 if (!parent_link_state) { 652 __pcie_aspm_config_link(link, state);
661 kfree(link_state);
662 goto unlock_out;
663 }
664 list_add(&link_state->link, &parent_link_state->children);
665 link_state->parent = parent_link_state;
666 }
667
668 pdev->link_state = link_state;
669
670 if (!blacklist) {
671 pcie_aspm_configure_common_clock(pdev);
672 pcie_aspm_cap_init(pdev);
673 } else { 653 } else {
674 link_state->enabled_state = PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1; 654 state = policy_to_aspm_state(link);
675 link_state->bios_aspm_state = 0; 655 __pcie_aspm_configure_link_state(link, state);
676 /* Set support state to 0, so we will disable ASPM later */
677 link_state->support_state = 0;
678 } 656 }
679 657
680 link_state->pdev = pdev; 658 /* Setup initial Clock PM state */
681 list_add(&link_state->sibiling, &link_list); 659 state = (link->clkpm_capable) ? policy_to_clkpm_state(link) : 0;
682 660 pcie_set_clkpm(link, state);
683 if (link_state->downstream_has_switch) { 661unlock:
684 /*
685 * If link has switch, delay the link config. The leaf link
686 * initialization will config the whole hierarchy. but we must
687 * make sure BIOS doesn't set unsupported link state
688 **/
689 state = pcie_aspm_check_state(pdev, link_state->bios_aspm_state);
690 __pcie_aspm_config_link(pdev, state);
691 } else
692 __pcie_aspm_configure_link_state(pdev,
693 policy_to_aspm_state(pdev));
694
695 pcie_check_clock_pm(pdev, blacklist);
696
697unlock_out:
698 if (error)
699 free_link_state(pdev);
700 mutex_unlock(&aspm_lock); 662 mutex_unlock(&aspm_lock);
701out: 663out:
702 up_read(&pci_bus_sem); 664 up_read(&pci_bus_sem);
@@ -725,11 +687,11 @@ void pcie_aspm_exit_link_state(struct pci_dev *pdev)
725 687
726 /* All functions are removed, so just disable ASPM for the link */ 688 /* All functions are removed, so just disable ASPM for the link */
727 __pcie_aspm_config_one_dev(parent, 0); 689 __pcie_aspm_config_one_dev(parent, 0);
728 list_del(&link_state->sibiling); 690 list_del(&link_state->sibling);
729 list_del(&link_state->link); 691 list_del(&link_state->link);
730 /* Clock PM is for endpoint device */ 692 /* Clock PM is for endpoint device */
731 693
732 free_link_state(parent); 694 free_link_state(link_state);
733out: 695out:
734 mutex_unlock(&aspm_lock); 696 mutex_unlock(&aspm_lock);
735 up_read(&pci_bus_sem); 697 up_read(&pci_bus_sem);
@@ -749,7 +711,7 @@ void pcie_aspm_pm_state_change(struct pci_dev *pdev)
749 * devices changed PM state, we should recheck if latency meets all 711 * devices changed PM state, we should recheck if latency meets all
750 * functions' requirement 712 * functions' requirement
751 */ 713 */
752 pcie_aspm_configure_link_state(pdev, link_state->enabled_state); 714 pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
753} 715}
754 716
755/* 717/*
@@ -772,14 +734,12 @@ void pci_disable_link_state(struct pci_dev *pdev, int state)
772 down_read(&pci_bus_sem); 734 down_read(&pci_bus_sem);
773 mutex_lock(&aspm_lock); 735 mutex_lock(&aspm_lock);
774 link_state = parent->link_state; 736 link_state = parent->link_state;
775 link_state->support_state &= 737 link_state->aspm_support &= ~state;
776 ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1)); 738 __pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
777 if (state & PCIE_LINK_STATE_CLKPM) 739 if (state & PCIE_LINK_STATE_CLKPM) {
778 link_state->clk_pm_capable = 0; 740 link_state->clkpm_capable = 0;
779 741 pcie_set_clkpm(link_state, 0);
780 __pcie_aspm_configure_link_state(parent, link_state->enabled_state); 742 }
781 if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
782 pcie_set_clock_pm(parent, 0);
783 mutex_unlock(&aspm_lock); 743 mutex_unlock(&aspm_lock);
784 up_read(&pci_bus_sem); 744 up_read(&pci_bus_sem);
785} 745}
@@ -788,7 +748,6 @@ EXPORT_SYMBOL(pci_disable_link_state);
788static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) 748static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
789{ 749{
790 int i; 750 int i;
791 struct pci_dev *pdev;
792 struct pcie_link_state *link_state; 751 struct pcie_link_state *link_state;
793 752
794 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 753 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
@@ -802,14 +761,10 @@ static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
802 down_read(&pci_bus_sem); 761 down_read(&pci_bus_sem);
803 mutex_lock(&aspm_lock); 762 mutex_lock(&aspm_lock);
804 aspm_policy = i; 763 aspm_policy = i;
805 list_for_each_entry(link_state, &link_list, sibiling) { 764 list_for_each_entry(link_state, &link_list, sibling) {
806 pdev = link_state->pdev; 765 __pcie_aspm_configure_link_state(link_state,
807 __pcie_aspm_configure_link_state(pdev, 766 policy_to_aspm_state(link_state));
808 policy_to_aspm_state(pdev)); 767 pcie_set_clkpm(link_state, policy_to_clkpm_state(link_state));
809 if (link_state->clk_pm_capable &&
810 link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
811 pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
812
813 } 768 }
814 mutex_unlock(&aspm_lock); 769 mutex_unlock(&aspm_lock);
815 up_read(&pci_bus_sem); 770 up_read(&pci_bus_sem);
@@ -838,7 +793,7 @@ static ssize_t link_state_show(struct device *dev,
838 struct pci_dev *pci_device = to_pci_dev(dev); 793 struct pci_dev *pci_device = to_pci_dev(dev);
839 struct pcie_link_state *link_state = pci_device->link_state; 794 struct pcie_link_state *link_state = pci_device->link_state;
840 795
841 return sprintf(buf, "%d\n", link_state->enabled_state); 796 return sprintf(buf, "%d\n", link_state->aspm_enabled);
842} 797}
843 798
844static ssize_t link_state_store(struct device *dev, 799static ssize_t link_state_store(struct device *dev,
@@ -846,7 +801,7 @@ static ssize_t link_state_store(struct device *dev,
846 const char *buf, 801 const char *buf,
847 size_t n) 802 size_t n)
848{ 803{
849 struct pci_dev *pci_device = to_pci_dev(dev); 804 struct pci_dev *pdev = to_pci_dev(dev);
850 int state; 805 int state;
851 806
852 if (n < 1) 807 if (n < 1)
@@ -854,7 +809,7 @@ static ssize_t link_state_store(struct device *dev,
854 state = buf[0]-'0'; 809 state = buf[0]-'0';
855 if (state >= 0 && state <= 3) { 810 if (state >= 0 && state <= 3) {
856 /* setup link aspm state */ 811 /* setup link aspm state */
857 pcie_aspm_configure_link_state(pci_device, state); 812 pcie_aspm_configure_link_state(pdev->link_state, state);
858 return n; 813 return n;
859 } 814 }
860 815
@@ -868,7 +823,7 @@ static ssize_t clk_ctl_show(struct device *dev,
868 struct pci_dev *pci_device = to_pci_dev(dev); 823 struct pci_dev *pci_device = to_pci_dev(dev);
869 struct pcie_link_state *link_state = pci_device->link_state; 824 struct pcie_link_state *link_state = pci_device->link_state;
870 825
871 return sprintf(buf, "%d\n", link_state->clk_pm_enabled); 826 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
872} 827}
873 828
874static ssize_t clk_ctl_store(struct device *dev, 829static ssize_t clk_ctl_store(struct device *dev,
@@ -876,7 +831,7 @@ static ssize_t clk_ctl_store(struct device *dev,
876 const char *buf, 831 const char *buf,
877 size_t n) 832 size_t n)
878{ 833{
879 struct pci_dev *pci_device = to_pci_dev(dev); 834 struct pci_dev *pdev = to_pci_dev(dev);
880 int state; 835 int state;
881 836
882 if (n < 1) 837 if (n < 1)
@@ -885,7 +840,7 @@ static ssize_t clk_ctl_store(struct device *dev,
885 840
886 down_read(&pci_bus_sem); 841 down_read(&pci_bus_sem);
887 mutex_lock(&aspm_lock); 842 mutex_lock(&aspm_lock);
888 pcie_set_clock_pm(pci_device, !!state); 843 pcie_set_clkpm_nocheck(pdev->link_state, !!state);
889 mutex_unlock(&aspm_lock); 844 mutex_unlock(&aspm_lock);
890 up_read(&pci_bus_sem); 845 up_read(&pci_bus_sem);
891 846
@@ -904,10 +859,10 @@ void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
904 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 859 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
905 return; 860 return;
906 861
907 if (link_state->support_state) 862 if (link_state->aspm_support)
908 sysfs_add_file_to_group(&pdev->dev.kobj, 863 sysfs_add_file_to_group(&pdev->dev.kobj,
909 &dev_attr_link_state.attr, power_group); 864 &dev_attr_link_state.attr, power_group);
910 if (link_state->clk_pm_capable) 865 if (link_state->clkpm_capable)
911 sysfs_add_file_to_group(&pdev->dev.kobj, 866 sysfs_add_file_to_group(&pdev->dev.kobj,
912 &dev_attr_clk_ctl.attr, power_group); 867 &dev_attr_clk_ctl.attr, power_group);
913} 868}
@@ -920,10 +875,10 @@ void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
920 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) 875 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
921 return; 876 return;
922 877
923 if (link_state->support_state) 878 if (link_state->aspm_support)
924 sysfs_remove_file_from_group(&pdev->dev.kobj, 879 sysfs_remove_file_from_group(&pdev->dev.kobj,
925 &dev_attr_link_state.attr, power_group); 880 &dev_attr_link_state.attr, power_group);
926 if (link_state->clk_pm_capable) 881 if (link_state->clkpm_capable)
927 sysfs_remove_file_from_group(&pdev->dev.kobj, 882 sysfs_remove_file_from_group(&pdev->dev.kobj,
928 &dev_attr_clk_ctl.attr, power_group); 883 &dev_attr_clk_ctl.attr, power_group);
929} 884}
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-30 19:32:32 -0400