diff options
Diffstat (limited to 'drivers/pci/pcie/aspm.c')
| -rw-r--r-- | drivers/pci/pcie/aspm.c | 310 |
1 files changed, 291 insertions, 19 deletions
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 17ac1dce3286..973472c23d89 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -30,8 +30,29 @@ #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ #define ASPM_STATE_L1 (4) /* L1 state */ +#define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */ +#define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */ +#define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */ +#define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */ +#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) +#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ + ASPM_STATE_L1_2_MASK) #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) -#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1) +#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ + ASPM_STATE_L1SS) + +/* + * When L1 substates are enabled, the LTR L1.2 threshold is a timing parameter + * that decides whether L1.1 or L1.2 is entered (Refer PCIe spec for details). + * Not sure is there is a way to "calculate" this on the fly, but maybe we + * could turn it into a parameter in future. This value has been taken from + * the following files from Intel's coreboot (which is the only code I found + * to have used this): + * https://www.coreboot.org/pipermail/coreboot-gerrit/2015-March/021134.html + * https://review.coreboot.org/#/c/8832/ + */ +#define LTR_L1_2_THRESHOLD_BITS ((1 << 21) | (1 << 23) | (1 << 30)) struct aspm_latency { u32 l0s; /* L0s latency (nsec) */ @@ -40,6 +61,7 @@ struct aspm_latency { struct pcie_link_state { struct pci_dev *pdev; /* Upstream component of the Link */ + struct pci_dev *downstream; /* Downstream component, function 0 */ struct pcie_link_state *root; /* pointer to the root port link */ struct pcie_link_state *parent; /* pointer to the parent Link state */ struct list_head sibling; /* node in link_list */ @@ -47,11 +69,11 @@ struct pcie_link_state { struct list_head link; /* node in parent's children list */ /* ASPM state */ - u32 aspm_support:3; /* Supported ASPM state */ - u32 aspm_enabled:3; /* Enabled ASPM state */ - u32 aspm_capable:3; /* Capable ASPM state with latency */ - u32 aspm_default:3; /* Default ASPM state by BIOS */ - u32 aspm_disable:3; /* Disabled ASPM state */ + u32 aspm_support:7; /* Supported ASPM state */ + u32 aspm_enabled:7; /* Enabled ASPM state */ + u32 aspm_capable:7; /* Capable ASPM state with latency */ + u32 aspm_default:7; /* Default ASPM state by BIOS */ + u32 aspm_disable:7; /* Disabled ASPM state */ /* Clock PM state */ u32 clkpm_capable:1; /* Clock PM capable? */ @@ -66,6 +88,14 @@ struct pcie_link_state { * has one slot under it, so at most there are 8 functions. */ struct aspm_latency acceptable[8]; + + /* L1 PM Substate info */ + struct { + u32 up_cap_ptr; /* L1SS cap ptr in upstream dev */ + u32 dw_cap_ptr; /* L1SS cap ptr in downstream dev */ + u32 ctl1; /* value to be programmed in ctl1 */ + u32 ctl2; /* value to be programmed in ctl2 */ + } l1ss; }; static int aspm_disabled, aspm_force; @@ -76,11 +106,14 @@ static LIST_HEAD(link_list); #define POLICY_DEFAULT 0 /* BIOS default setting */ #define POLICY_PERFORMANCE 1 /* high performance */ #define POLICY_POWERSAVE 2 /* high power saving */ +#define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */ #ifdef CONFIG_PCIEASPM_PERFORMANCE static int aspm_policy = POLICY_PERFORMANCE; #elif defined CONFIG_PCIEASPM_POWERSAVE static int aspm_policy = POLICY_POWERSAVE; +#elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE +static int aspm_policy = POLICY_POWER_SUPERSAVE; #else static int aspm_policy; #endif @@ -88,7 +121,8 @@ static int aspm_policy; static const char *policy_str[] = { [POLICY_DEFAULT] = "default", [POLICY_PERFORMANCE] = "performance", - [POLICY_POWERSAVE] = "powersave" + [POLICY_POWERSAVE] = "powersave", + [POLICY_POWER_SUPERSAVE] = "powersupersave" }; #define LINK_RETRAIN_TIMEOUT HZ @@ -101,6 +135,9 @@ static int policy_to_aspm_state(struct pcie_link_state *link) return 0; case POLICY_POWERSAVE: /* Enable ASPM L0s/L1 */ + return (ASPM_STATE_L0S | ASPM_STATE_L1); + case POLICY_POWER_SUPERSAVE: + /* Enable Everything */ return ASPM_STATE_ALL; case POLICY_DEFAULT: return link->aspm_default; @@ -115,7 +152,8 @@ static int policy_to_clkpm_state(struct pcie_link_state *link) /* Disable ASPM and Clock PM */ return 0; case POLICY_POWERSAVE: - /* Disable Clock PM */ + case POLICY_POWER_SUPERSAVE: + /* Enable Clock PM */ return 1; case POLICY_DEFAULT: return link->clkpm_default; @@ -278,11 +316,33 @@ static u32 calc_l1_acceptable(u32 encoding) return (1000 << encoding); } +/* Convert L1SS T_pwr encoding to usec */ +static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) +{ + switch (scale) { + case 0: + return val * 2; + case 1: + return val * 10; + case 2: + return val * 100; + } + dev_err(&pdev->dev, "%s: Invalid T_PwrOn scale: %u\n", + __func__, scale); + return 0; +} + struct aspm_register_info { u32 support:2; u32 enabled:2; u32 latency_encoding_l0s; u32 latency_encoding_l1; + + /* L1 substates */ + u32 l1ss_cap_ptr; + u32 l1ss_cap; + u32 l1ss_ctl1; + u32 l1ss_ctl2; }; static void pcie_get_aspm_reg(struct pci_dev *pdev, @@ -297,6 +357,22 @@ static void pcie_get_aspm_reg(struct pci_dev *pdev, info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; + + /* Read L1 PM substate capabilities */ + info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0; + info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); + if (!info->l1ss_cap_ptr) + return; + pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP, + &info->l1ss_cap); + if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) { + info->l1ss_cap = 0; + return; + } + pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1, + &info->l1ss_ctl1); + pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2, + &info->l1ss_ctl2); } static void pcie_aspm_check_latency(struct pci_dev *endpoint) @@ -327,6 +403,14 @@ static void pcie_aspm_check_latency(struct pci_dev *endpoint) * Check L1 latency. * Every switch on the path to root complex need 1 * more microsecond for L1. Spec doesn't mention L0s. + * + * The exit latencies for L1 substates are not advertised + * by a device. Since the spec also doesn't mention a way + * to determine max latencies introduced by enabling L1 + * substates on the components, it is not clear how to do + * a L1 substate exit latency check. We assume that the + * L1 exit latencies advertised by a device include L1 + * substate latencies (and hence do not do any check). */ latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); if ((link->aspm_capable & ASPM_STATE_L1) && @@ -338,6 +422,60 @@ static void pcie_aspm_check_latency(struct pci_dev *endpoint) } } +/* + * The L1 PM substate capability is only implemented in function 0 in a + * multi function device. + */ +static struct pci_dev *pci_function_0(struct pci_bus *linkbus) +{ + struct pci_dev *child; + + list_for_each_entry(child, &linkbus->devices, bus_list) + if (PCI_FUNC(child->devfn) == 0) + return child; + return NULL; +} + +/* Calculate L1.2 PM substate timing parameters */ +static void aspm_calc_l1ss_info(struct pcie_link_state *link, + struct aspm_register_info *upreg, + struct aspm_register_info *dwreg) +{ + u32 val1, val2, scale1, scale2; + + link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr; + link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr; + link->l1ss.ctl1 = link->l1ss.ctl2 = 0; + + if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) + return; + + /* Choose the greater of the two T_cmn_mode_rstr_time */ + val1 = (upreg->l1ss_cap >> 8) & 0xFF; + val2 = (upreg->l1ss_cap >> 8) & 0xFF; + if (val1 > val2) + link->l1ss.ctl1 |= val1 << 8; + else + link->l1ss.ctl1 |= val2 << 8; + /* + * We currently use LTR L1.2 threshold to be fixed constant picked from + * Intel's coreboot. + */ + link->l1ss.ctl1 |= LTR_L1_2_THRESHOLD_BITS; + + /* Choose the greater of the two T_pwr_on */ + val1 = (upreg->l1ss_cap >> 19) & 0x1F; + scale1 = (upreg->l1ss_cap >> 16) & 0x03; + val2 = (dwreg->l1ss_cap >> 19) & 0x1F; + scale2 = (dwreg->l1ss_cap >> 16) & 0x03; + + if (calc_l1ss_pwron(link->pdev, scale1, val1) > + calc_l1ss_pwron(link->downstream, scale2, val2)) + link->l1ss.ctl2 |= scale1 | (val1 << 3); + else + link->l1ss.ctl2 |= scale2 | (val2 << 3); +} + static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) { struct pci_dev *child, *parent = link->pdev; @@ -353,8 +491,9 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Get upstream/downstream components' register state */ pcie_get_aspm_reg(parent, &upreg); - child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); + child = pci_function_0(linkbus); pcie_get_aspm_reg(child, &dwreg); + link->downstream = child; /* * If ASPM not supported, don't mess with the clocks and link, @@ -397,6 +536,28 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); + /* Setup L1 substate */ + if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) + link->aspm_support |= ASPM_STATE_L1_1; + if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) + link->aspm_support |= ASPM_STATE_L1_2; + if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) + link->aspm_support |= ASPM_STATE_L1_1_PCIPM; + if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) + link->aspm_support |= ASPM_STATE_L1_2_PCIPM; + + if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) + link->aspm_enabled |= ASPM_STATE_L1_1; + if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) + link->aspm_enabled |= ASPM_STATE_L1_2; + if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) + link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; + if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) + link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; + + if (link->aspm_support & ASPM_STATE_L1SS) + aspm_calc_l1ss_info(link, &upreg, &dwreg); + /* Save default state */ link->aspm_default = link->aspm_enabled; @@ -435,6 +596,92 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) } } +static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, + u32 clear, u32 set) +{ + u32 val; + + pci_read_config_dword(pdev, pos, &val); + val &= ~clear; + val |= set; + pci_write_config_dword(pdev, pos, val); +} + +/* Configure the ASPM L1 substates */ +static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) +{ + u32 val, enable_req; + struct pci_dev *child = link->downstream, *parent = link->pdev; + u32 up_cap_ptr = link->l1ss.up_cap_ptr; + u32 dw_cap_ptr = link->l1ss.dw_cap_ptr; + + enable_req = (link->aspm_enabled ^ state) & state; + + /* + * Here are the rules specified in the PCIe spec for enabling L1SS: + * - When enabling L1.x, enable bit at parent first, then at child + * - When disabling L1.x, disable bit at child first, then at parent + * - When enabling ASPM L1.x, need to disable L1 + * (at child followed by parent). + * - The ASPM/PCIPM L1.2 must be disabled while programming timing + * parameters + * + * To keep it simple, disable all L1SS bits first, and later enable + * what is needed. + */ + + /* Disable all L1 substates */ + pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_L1SS_MASK, 0); + pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, + PCI_L1SS_CTL1_L1SS_MASK, 0); + /* + * If needed, disable L1, and it gets enabled later + * in pcie_config_aspm_link(). + */ + if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) { + pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPM_L1, 0); + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPM_L1, 0); + } + + if (enable_req & ASPM_STATE_L1_2_MASK) { + + /* Program T_pwr_on in both ports */ + pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2, + link->l1ss.ctl2); + pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2, + link->l1ss.ctl2); + + /* Program T_cmn_mode in parent */ + pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, + 0xFF00, link->l1ss.ctl1); + + /* Program LTR L1.2 threshold in both ports */ + pci_clear_and_set_dword(parent, dw_cap_ptr + PCI_L1SS_CTL1, + 0xE3FF0000, link->l1ss.ctl1); + pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, + 0xE3FF0000, link->l1ss.ctl1); + } + + val = 0; + if (state & ASPM_STATE_L1_1) + val |= PCI_L1SS_CTL1_ASPM_L1_1; + if (state & ASPM_STATE_L1_2) + val |= PCI_L1SS_CTL1_ASPM_L1_2; + if (state & ASPM_STATE_L1_1_PCIPM) + val |= PCI_L1SS_CTL1_PCIPM_L1_1; + if (state & ASPM_STATE_L1_2_PCIPM) + val |= PCI_L1SS_CTL1_PCIPM_L1_2; + + /* Enable what we need to enable */ + pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, + PCI_L1SS_CAP_L1_PM_SS, val); + pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, + PCI_L1SS_CAP_L1_PM_SS, val); +} + static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) { pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, @@ -444,11 +691,23 @@ static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) { u32 upstream = 0, dwstream = 0; - struct pci_dev *child, *parent = link->pdev; + struct pci_dev *child = link->downstream, *parent = link->pdev; struct pci_bus *linkbus = parent->subordinate; - /* Nothing to do if the link is already in the requested state */ + /* Enable only the states that were not explicitly disabled */ state &= (link->aspm_capable & ~link->aspm_disable); + + /* Can't enable any substates if L1 is not enabled */ + if (!(state & ASPM_STATE_L1)) + state &= ~ASPM_STATE_L1SS; + + /* Spec says both ports must be in D0 before enabling PCI PM substates*/ + if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { + state &= ~ASPM_STATE_L1_SS_PCIPM; + state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); + } + + /* Nothing to do if the link is already in the requested state */ if (link->aspm_enabled == state) return; /* Convert ASPM state to upstream/downstream ASPM register state */ @@ -460,6 +719,10 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) upstream |= PCI_EXP_LNKCTL_ASPM_L1; dwstream |= PCI_EXP_LNKCTL_ASPM_L1; } + + if (link->aspm_capable & ASPM_STATE_L1SS) + pcie_config_aspm_l1ss(link, state); + /* * Spec 2.0 suggests all functions should be configured the * same setting for ASPM. Enabling ASPM L1 should be done in @@ -532,25 +795,32 @@ static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) link = kzalloc(sizeof(*link), GFP_KERNEL); if (!link) return NULL; + INIT_LIST_HEAD(&link->sibling); INIT_LIST_HEAD(&link->children); INIT_LIST_HEAD(&link->link); link->pdev = pdev; - if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) { + + /* + * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe + * hierarchies. + */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE) { + link->root = link; + } else { struct pcie_link_state *parent; + parent = pdev->bus->parent->self->link_state; if (!parent) { kfree(link); return NULL; } + link->parent = parent; + link->root = link->parent->root; list_add(&link->link, &parent->children); } - /* Setup a pointer to the root port link */ - if (!link->parent) - link->root = link; - else - link->root = link->parent->root; list_add(&link->sibling, &link_list); pdev->link_state = link; @@ -612,7 +882,8 @@ void pcie_aspm_init_link_state(struct pci_dev *pdev) * the BIOS's expectation, we'll do so once pci_enable_device() is * called. */ - if (aspm_policy != POLICY_POWERSAVE) { + if (aspm_policy != POLICY_POWERSAVE && + aspm_policy != POLICY_POWER_SUPERSAVE) { pcie_config_aspm_path(link); pcie_set_clkpm(link, policy_to_clkpm_state(link)); } @@ -712,7 +983,8 @@ void pcie_aspm_powersave_config_link(struct pci_dev *pdev) if (aspm_disabled || !link) return; - if (aspm_policy != POLICY_POWERSAVE) + if (aspm_policy != POLICY_POWERSAVE && + aspm_policy != POLICY_POWER_SUPERSAVE) return; down_read(&pci_bus_sem); |