diff options
Diffstat (limited to 'drivers/scsi/ufs/ufshcd.c')
| -rw-r--r-- | drivers/scsi/ufs/ufshcd.c | 1588 |
1 files changed, 1342 insertions, 246 deletions
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 20e5e5fb048c..dc6efbd1be8e 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -45,6 +45,9 @@ #include "ufs_quirks.h" #include "unipro.h" +#define CREATE_TRACE_POINTS +#include <trace/events/ufs.h> + #define UFSHCD_REQ_SENSE_SIZE 18 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ @@ -94,6 +97,9 @@ _ret; \ }) +#define ufshcd_hex_dump(prefix_str, buf, len) \ +print_hex_dump(KERN_ERR, prefix_str, DUMP_PREFIX_OFFSET, 16, 4, buf, len, false) + static u32 ufs_query_desc_max_size[] = { QUERY_DESC_DEVICE_MAX_SIZE, QUERY_DESC_CONFIGURAION_MAX_SIZE, @@ -185,6 +191,22 @@ ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl) return ufs_pm_lvl_states[lvl].link_state; } +static inline enum ufs_pm_level +ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, + enum uic_link_state link_state) +{ + enum ufs_pm_level lvl; + + for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) { + if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) && + (ufs_pm_lvl_states[lvl].link_state == link_state)) + return lvl; + } + + /* if no match found, return the level 0 */ + return UFS_PM_LVL_0; +} + static struct ufs_dev_fix ufs_fixups[] = { /* UFS cards deviations table */ UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, @@ -212,6 +234,7 @@ static struct ufs_dev_fix ufs_fixups[] = { static void ufshcd_tmc_handler(struct ufs_hba *hba); static void ufshcd_async_scan(void *data, async_cookie_t cookie); static int ufshcd_reset_and_restore(struct ufs_hba *hba); +static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); static void ufshcd_hba_exit(struct ufs_hba *hba); static int ufshcd_probe_hba(struct ufs_hba *hba); @@ -223,6 +246,10 @@ static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba); static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); +static void ufshcd_resume_clkscaling(struct ufs_hba *hba); +static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); +static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba); +static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up); static irqreturn_t ufshcd_intr(int irq, void *__hba); static int ufshcd_config_pwr_mode(struct ufs_hba *hba, struct ufs_pa_layer_attr *desired_pwr_mode); @@ -267,6 +294,214 @@ static inline void ufshcd_remove_non_printable(char *val) *val = ' '; } +static void ufshcd_add_command_trace(struct ufs_hba *hba, + unsigned int tag, const char *str) +{ + sector_t lba = -1; + u8 opcode = 0; + u32 intr, doorbell; + struct ufshcd_lrb *lrbp; + int transfer_len = -1; + + if (!trace_ufshcd_command_enabled()) + return; + + lrbp = &hba->lrb[tag]; + + if (lrbp->cmd) { /* data phase exists */ + opcode = (u8)(*lrbp->cmd->cmnd); + if ((opcode == READ_10) || (opcode == WRITE_10)) { + /* + * Currently we only fully trace read(10) and write(10) + * commands + */ + if (lrbp->cmd->request && lrbp->cmd->request->bio) + lba = + lrbp->cmd->request->bio->bi_iter.bi_sector; + transfer_len = be32_to_cpu( + lrbp->ucd_req_ptr->sc.exp_data_transfer_len); + } + } + + intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); + doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); + trace_ufshcd_command(dev_name(hba->dev), str, tag, + doorbell, transfer_len, intr, lba, opcode); +} + +static void ufshcd_print_clk_freqs(struct ufs_hba *hba) +{ + struct ufs_clk_info *clki; + struct list_head *head = &hba->clk_list_head; + + if (!head || list_empty(head)) + return; + + list_for_each_entry(clki, head, list) { + if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq && + clki->max_freq) + dev_err(hba->dev, "clk: %s, rate: %u\n", + clki->name, clki->curr_freq); + } +} + +static void ufshcd_print_uic_err_hist(struct ufs_hba *hba, + struct ufs_uic_err_reg_hist *err_hist, char *err_name) +{ + int i; + + for (i = 0; i < UIC_ERR_REG_HIST_LENGTH; i++) { + int p = (i + err_hist->pos - 1) % UIC_ERR_REG_HIST_LENGTH; + + if (err_hist->reg[p] == 0) + continue; + dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, i, + err_hist->reg[p], ktime_to_us(err_hist->tstamp[p])); + } +} + +static void ufshcd_print_host_regs(struct ufs_hba *hba) +{ + /* + * hex_dump reads its data without the readl macro. This might + * cause inconsistency issues on some platform, as the printed + * values may be from cache and not the most recent value. + * To know whether you are looking at an un-cached version verify + * that IORESOURCE_MEM flag is on when xxx_get_resource() is invoked + * during platform/pci probe function. + */ + ufshcd_hex_dump("host regs: ", hba->mmio_base, UFSHCI_REG_SPACE_SIZE); + dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n", + hba->ufs_version, hba->capabilities); + dev_err(hba->dev, + "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n", + (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks); + dev_err(hba->dev, + "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n", + ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp), + hba->ufs_stats.hibern8_exit_cnt); + + ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err"); + ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err"); + ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err"); + ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err"); + ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err"); + + ufshcd_print_clk_freqs(hba); + + if (hba->vops && hba->vops->dbg_register_dump) + hba->vops->dbg_register_dump(hba); +} + +static +void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt) +{ + struct ufshcd_lrb *lrbp; + int prdt_length; + int tag; + + for_each_set_bit(tag, &bitmap, hba->nutrs) { + lrbp = &hba->lrb[tag]; + + dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", + tag, ktime_to_us(lrbp->issue_time_stamp)); + dev_err(hba->dev, + "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", + tag, (u64)lrbp->utrd_dma_addr); + + ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, + sizeof(struct utp_transfer_req_desc)); + dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, + (u64)lrbp->ucd_req_dma_addr); + ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, + sizeof(struct utp_upiu_req)); + dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, + (u64)lrbp->ucd_rsp_dma_addr); + ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, + sizeof(struct utp_upiu_rsp)); + + prdt_length = le16_to_cpu( + lrbp->utr_descriptor_ptr->prd_table_length); + dev_err(hba->dev, + "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", + tag, prdt_length, + (u64)lrbp->ucd_prdt_dma_addr); + + if (pr_prdt) + ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, + sizeof(struct ufshcd_sg_entry) * prdt_length); + } +} + +static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap) +{ + struct utp_task_req_desc *tmrdp; + int tag; + + for_each_set_bit(tag, &bitmap, hba->nutmrs) { + tmrdp = &hba->utmrdl_base_addr[tag]; + dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag); + ufshcd_hex_dump("TM TRD: ", &tmrdp->header, + sizeof(struct request_desc_header)); + dev_err(hba->dev, "TM[%d] - Task Management Request UPIU\n", + tag); + ufshcd_hex_dump("TM REQ: ", tmrdp->task_req_upiu, + sizeof(struct utp_upiu_req)); + dev_err(hba->dev, "TM[%d] - Task Management Response UPIU\n", + tag); + ufshcd_hex_dump("TM RSP: ", tmrdp->task_rsp_upiu, + sizeof(struct utp_task_req_desc)); + } +} + +static void ufshcd_print_host_state(struct ufs_hba *hba) +{ + dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); + dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n", + hba->lrb_in_use, hba->outstanding_tasks, hba->outstanding_reqs); + dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", + hba->saved_err, hba->saved_uic_err); + dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", + hba->curr_dev_pwr_mode, hba->uic_link_state); + dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n", + hba->pm_op_in_progress, hba->is_sys_suspended); + dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n", + hba->auto_bkops_enabled, hba->host->host_self_blocked); + dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); + dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", + hba->eh_flags, hba->req_abort_count); + dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n", + hba->capabilities, hba->caps); + dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks, + hba->dev_quirks); +} + +/** + * ufshcd_print_pwr_info - print power params as saved in hba + * power info + * @hba: per-adapter instance + */ +static void ufshcd_print_pwr_info(struct ufs_hba *hba) +{ + static const char * const names[] = { + "INVALID MODE", + "FAST MODE", + "SLOW_MODE", + "INVALID MODE", + "FASTAUTO_MODE", + "SLOWAUTO_MODE", + "INVALID MODE", + }; + + dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n", + __func__, + hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, + hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, + names[hba->pwr_info.pwr_rx], + names[hba->pwr_info.pwr_tx], + hba->pwr_info.hs_rate); +} + /* * ufshcd_wait_for_register - wait for register value to change * @hba - per-adapter interface @@ -605,6 +840,28 @@ static inline int ufshcd_is_hba_active(struct ufs_hba *hba) return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & 0x1) ? 0 : 1; } +static const char *ufschd_uic_link_state_to_string( + enum uic_link_state state) +{ + switch (state) { + case UIC_LINK_OFF_STATE: return "OFF"; + case UIC_LINK_ACTIVE_STATE: return "ACTIVE"; + case UIC_LINK_HIBERN8_STATE: return "HIBERN8"; + default: return "UNKNOWN"; + } +} + +static const char *ufschd_ufs_dev_pwr_mode_to_string( + enum ufs_dev_pwr_mode state) +{ + switch (state) { + case UFS_ACTIVE_PWR_MODE: return "ACTIVE"; + case UFS_SLEEP_PWR_MODE: return "SLEEP"; + case UFS_POWERDOWN_PWR_MODE: return "POWERDOWN"; + default: return "UNKNOWN"; + } +} + u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba) { /* HCI version 1.0 and 1.1 supports UniPro 1.41 */ @@ -633,20 +890,523 @@ static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba) return false; } +static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) +{ + int ret = 0; + struct ufs_clk_info *clki; + struct list_head *head = &hba->clk_list_head; + ktime_t start = ktime_get(); + bool clk_state_changed = false; + + if (!head || list_empty(head)) + goto out; + + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE); + if (ret) + return ret; + + list_for_each_entry(clki, head, list) { + if (!IS_ERR_OR_NULL(clki->clk)) { + if (scale_up && clki->max_freq) { + if (clki->curr_freq == clki->max_freq) + continue; + + clk_state_changed = true; + ret = clk_set_rate(clki->clk, clki->max_freq); + if (ret) { + dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", + __func__, clki->name, + clki->max_freq, ret); + break; + } + trace_ufshcd_clk_scaling(dev_name(hba->dev), + "scaled up", clki->name, + clki->curr_freq, + clki->max_freq); + + clki->curr_freq = clki->max_freq; + + } else if (!scale_up && clki->min_freq) { + if (clki->curr_freq == clki->min_freq) + continue; + + clk_state_changed = true; + ret = clk_set_rate(clki->clk, clki->min_freq); + if (ret) { + dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", + __func__, clki->name, + clki->min_freq, ret); + break; + } + trace_ufshcd_clk_scaling(dev_name(hba->dev), + "scaled down", clki->name, + clki->curr_freq, + clki->min_freq); + clki->curr_freq = clki->min_freq; + } + } + dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, + clki->name, clk_get_rate(clki->clk)); + } + + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE); + +out: + if (clk_state_changed) + trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + (scale_up ? "up" : "down"), + ktime_to_us(ktime_sub(ktime_get(), start)), ret); + return ret; +} + +/** + * ufshcd_is_devfreq_scaling_required - check if scaling is required or not + * @hba: per adapter instance + * @scale_up: True if scaling up and false if scaling down + * + * Returns true if scaling is required, false otherwise. + */ +static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, + bool scale_up) +{ + struct ufs_clk_info *clki; + struct list_head *head = &hba->clk_list_head; + + if (!head || list_empty(head)) + return false; + + list_for_each_entry(clki, head, list) { + if (!IS_ERR_OR_NULL(clki->clk)) { + if (scale_up && clki->max_freq) { + if (clki->curr_freq == clki->max_freq) + continue; + return true; + } else if (!scale_up && clki->min_freq) { + if (clki->curr_freq == clki->min_freq) + continue; + return true; + } + } + } + + return false; +} + +static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, + u64 wait_timeout_us) +{ + unsigned long flags; + int ret = 0; + u32 tm_doorbell; + u32 tr_doorbell; + bool timeout = false, do_last_check = false; + ktime_t start; + + ufshcd_hold(hba, false); + spin_lock_irqsave(hba->host->host_lock, flags); + /* + * Wait for all the outstanding tasks/transfer requests. + * Verify by checking the doorbell registers are clear. + */ + start = ktime_get(); + do { + if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) { + ret = -EBUSY; + goto out; + } + + tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL); + tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); + if (!tm_doorbell && !tr_doorbell) { + timeout = false; + break; + } else if (do_last_check) { + break; + } + + spin_unlock_irqrestore(hba->host->host_lock, flags); + schedule(); + if (ktime_to_us(ktime_sub(ktime_get(), start)) > + wait_timeout_us) { + timeout = true; + /* + * We might have scheduled out for long time so make + * sure to check if doorbells are cleared by this time + * or not. + */ + do_last_check = true; + } + spin_lock_irqsave(hba->host->host_lock, flags); + } while (tm_doorbell || tr_doorbell); + + if (timeout) { + dev_err(hba->dev, + "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n", + __func__, tm_doorbell, tr_doorbell); + ret = -EBUSY; + } +out: + spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_release(hba); + return ret; +} + +/** + * ufshcd_scale_gear - scale up/down UFS gear + * @hba: per adapter instance + * @scale_up: True for scaling up gear and false for scaling down + * + * Returns 0 for success, + * Returns -EBUSY if scaling can't happen at this time + * Returns non-zero for any other errors + */ +static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) +{ + #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1 + int ret = 0; + struct ufs_pa_layer_attr new_pwr_info; + + if (scale_up) { + memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info, + sizeof(struct ufs_pa_layer_attr)); + } else { + memcpy(&new_pwr_info, &hba->pwr_info, + sizeof(struct ufs_pa_layer_attr)); + + if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN + || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) { + /* save the current power mode */ + memcpy(&hba->clk_scaling.saved_pwr_info.info, + &hba->pwr_info, + sizeof(struct ufs_pa_layer_attr)); + + /* scale down gear */ + new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN; + new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN; + } + } + + /* check if the power mode needs to be changed or not? */ + ret = ufshcd_change_power_mode(hba, &new_pwr_info); + + if (ret) + dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)", + __func__, ret, + hba->pwr_info.gear_tx, hba->pwr_info.gear_rx, + new_pwr_info.gear_tx, new_pwr_info.gear_rx); + + return ret; +} + +static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba) +{ + #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */ + int ret = 0; + /* + * make sure that there are no outstanding requests when + * clock scaling is in progress + */ + scsi_block_requests(hba->host); + down_write(&hba->clk_scaling_lock); + if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) { + ret = -EBUSY; + up_write(&hba->clk_scaling_lock); + scsi_unblock_requests(hba->host); + } + + return ret; +} + +static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba) +{ + up_write(&hba->clk_scaling_lock); + scsi_unblock_requests(hba->host); +} + +/** + * ufshcd_devfreq_scale - scale up/down UFS clocks and gear + * @hba: per adapter instance + * @scale_up: True for scaling up and false for scalin down + * + * Returns 0 for success, + * Returns -EBUSY if scaling can't happen at this time + * Returns non-zero for any other errors + */ +static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up) +{ + int ret = 0; + + /* let's not get into low power until clock scaling is completed */ + ufshcd_hold(hba, false); + + ret = ufshcd_clock_scaling_prepare(hba); + if (ret) + return ret; + + /* scale down the gear before scaling down clocks */ + if (!scale_up) { + ret = ufshcd_scale_gear(hba, false); + if (ret) + goto out; + } + + ret = ufshcd_scale_clks(hba, scale_up); + if (ret) { + if (!scale_up) + ufshcd_scale_gear(hba, true); + goto out; + } + + /* scale up the gear after scaling up clocks */ + if (scale_up) { + ret = ufshcd_scale_gear(hba, true); + if (ret) { + ufshcd_scale_clks(hba, false); + goto out; + } + } + + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE); + +out: + ufshcd_clock_scaling_unprepare(hba); + ufshcd_release(hba); + return ret; +} + +static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) +{ + struct ufs_hba *hba = container_of(work, struct ufs_hba, + clk_scaling.suspend_work); + unsigned long irq_flags; + + spin_lock_irqsave(hba->host->host_lock, irq_flags); + if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) { + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + return; + } + hba->clk_scaling.is_suspended = true; + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + + __ufshcd_suspend_clkscaling(hba); +} + +static void ufshcd_clk_scaling_resume_work(struct work_struct *work) +{ + struct ufs_hba *hba = container_of(work, struct ufs_hba, + clk_scaling.resume_work); + unsigned long irq_flags; + + spin_lock_irqsave(hba->host->host_lock, irq_flags); + if (!hba->clk_scaling.is_suspended) { + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + return; + } + hba->clk_scaling.is_suspended = false; + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + + devfreq_resume_device(hba->devfreq); +} + +static int ufshcd_devfreq_target(struct device *dev, + unsigned long *freq, u32 flags) +{ + int ret = 0; + struct ufs_hba *hba = dev_get_drvdata(dev); + ktime_t start; + bool scale_up, sched_clk_scaling_suspend_work = false; + unsigned long irq_flags; + + if (!ufshcd_is_clkscaling_supported(hba)) + return -EINVAL; + + if ((*freq > 0) && (*freq < UINT_MAX)) { + dev_err(hba->dev, "%s: invalid freq = %lu\n", __func__, *freq); + return -EINVAL; + } + + spin_lock_irqsave(hba->host->host_lock, irq_flags); + if (ufshcd_eh_in_progress(hba)) { + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + return 0; + } + + if (!hba->clk_scaling.active_reqs) + sched_clk_scaling_suspend_work = true; + + scale_up = (*freq == UINT_MAX) ? true : false; + if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) { + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + ret = 0; + goto out; /* no state change required */ + } + spin_unlock_irqrestore(hba->host->host_lock, irq_flags); + + start = ktime_get(); + ret = ufshcd_devfreq_scale(hba, scale_up); + + trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + (scale_up ? "up" : "down"), + ktime_to_us(ktime_sub(ktime_get(), start)), ret); + +out: + if (sched_clk_scaling_suspend_work) + queue_work(hba->clk_scaling.workq, + &hba->clk_scaling.suspend_work); + + return ret; +} + + +static int ufshcd_devfreq_get_dev_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_clk_scaling *scaling = &hba->clk_scaling; + unsigned long flags; + + if (!ufshcd_is_clkscaling_supported(hba)) + return -EINVAL; + + memset(stat, 0, sizeof(*stat)); + + spin_lock_irqsave(hba->host->host_lock, flags); + if (!scaling->window_start_t) + goto start_window; + + if (scaling->is_busy_started) + scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), + scaling->busy_start_t)); + + stat->total_time = jiffies_to_usecs((long)jiffies - + (long)scaling->window_start_t); + stat->busy_time = scaling->tot_busy_t; +start_window: + scaling->window_start_t = jiffies; + scaling->tot_busy_t = 0; + + if (hba->outstanding_reqs) { + scaling->busy_start_t = ktime_get(); + scaling->is_busy_started = true; + } else { + scaling->busy_start_t = 0; + scaling->is_busy_started = false; + } + spin_unlock_irqrestore(hba->host->host_lock, flags); + return 0; +} + +static struct devfreq_dev_profile ufs_devfreq_profile = { + .polling_ms = 100, + .target = ufshcd_devfreq_target, + .get_dev_status = ufshcd_devfreq_get_dev_status, +}; + +static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba) +{ + unsigned long flags; + + devfreq_suspend_device(hba->devfreq); + spin_lock_irqsave(hba->host->host_lock, flags); + hba->clk_scaling.window_start_t = 0; + spin_unlock_irqrestore(hba->host->host_lock, flags); +} + static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) { - if (ufshcd_is_clkscaling_enabled(hba)) { - devfreq_suspend_device(hba->devfreq); - hba->clk_scaling.window_start_t = 0; + unsigned long flags; + bool suspend = false; + + if (!ufshcd_is_clkscaling_supported(hba)) + return; + + spin_lock_irqsave(hba->host->host_lock, flags); + if (!hba->clk_scaling.is_suspended) { + suspend = true; + hba->clk_scaling.is_suspended = true; } + spin_unlock_irqrestore(hba->host->host_lock, flags); + + if (suspend) + __ufshcd_suspend_clkscaling(hba); } static void ufshcd_resume_clkscaling(struct ufs_hba *hba) { - if (ufshcd_is_clkscaling_enabled(hba)) + unsigned long flags; + bool resume = false; + + if (!ufshcd_is_clkscaling_supported(hba)) + return; + + spin_lock_irqsave(hba->host->host_lock, flags); + if (hba->clk_scaling.is_suspended) { + resume = true; + hba->clk_scaling.is_suspended = false; + } + spin_unlock_irqrestore(hba->host->host_lock, flags); + + if (resume) devfreq_resume_device(hba->devfreq); } +static ssize_t ufshcd_clkscale_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed); +} + +static ssize_t ufshcd_clkscale_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + u32 value; + int err; + + if (kstrtou32(buf, 0, &value)) + return -EINVAL; + + value = !!value; + if (value == hba->clk_scaling.is_allowed) + goto out; + + pm_runtime_get_sync(hba->dev); + ufshcd_hold(hba, false); + + cancel_work_sync(&hba->clk_scaling.suspend_work); + cancel_work_sync(&hba->clk_scaling.resume_work); + + hba->clk_scaling.is_allowed = value; + + if (value) { + ufshcd_resume_clkscaling(hba); + } else { + ufshcd_suspend_clkscaling(hba); + err = ufshcd_devfreq_scale(hba, true); + if (err) + dev_err(hba->dev, "%s: failed to scale clocks up %d\n", + __func__, err); + } + + ufshcd_release(hba); + pm_runtime_put_sync(hba->dev); +out: + return count; +} + +static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba) +{ + hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show; + hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store; + sysfs_attr_init(&hba->clk_scaling.enable_attr.attr); + hba->clk_scaling.enable_attr.attr.name = "clkscale_enable"; + hba->clk_scaling.enable_attr.attr.mode = 0644; + if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr)) + dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n"); +} + static void ufshcd_ungate_work(struct work_struct *work) { int ret; @@ -680,7 +1440,6 @@ static void ufshcd_ungate_work(struct work_struct *work) hba->clk_gating.is_suspended = false; } unblock_reqs: - ufshcd_resume_clkscaling(hba); scsi_unblock_requests(hba->host); } @@ -727,6 +1486,8 @@ start: case REQ_CLKS_OFF: if (cancel_delayed_work(&hba->clk_gating.gate_work)) { hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(dev_name(hba->dev), + hba->clk_gating.state); break; } /* @@ -737,6 +1498,8 @@ start: case CLKS_OFF: scsi_block_requests(hba->host); hba->clk_gating.state = REQ_CLKS_ON; + trace_ufshcd_clk_gating(dev_name(hba->dev), + hba->clk_gating.state); schedule_work(&hba->clk_gating.ungate_work); /* * fall through to check if we should wait for this @@ -781,6 +1544,8 @@ static void ufshcd_gate_work(struct work_struct *work) if (hba->clk_gating.is_suspended || (hba->clk_gating.state == REQ_CLKS_ON)) { hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(dev_name(hba->dev), + hba->clk_gating.state); goto rel_lock; } @@ -796,13 +1561,13 @@ static void ufshcd_gate_work(struct work_struct *work) if (ufshcd_can_hibern8_during_gating(hba)) { if (ufshcd_uic_hibern8_enter(hba)) { hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(dev_name(hba->dev), + hba->clk_gating.state); goto out; } ufshcd_set_link_hibern8(hba); } - ufshcd_suspend_clkscaling(hba); - if (!ufshcd_is_link_active(hba)) ufshcd_setup_clocks(hba, false); else @@ -819,9 +1584,11 @@ static void ufshcd_gate_work(struct work_struct *work) * new requests arriving before the current cancel work is done. */ spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_gating.state == REQ_CLKS_OFF) + if (hba->clk_gating.state == REQ_CLKS_OFF) { hba->clk_gating.state = CLKS_OFF; - + trace_ufshcd_clk_gating(dev_name(hba->dev), + hba->clk_gating.state); + } rel_lock: spin_unlock_irqrestore(hba->host->host_lock, flags); out: @@ -844,6 +1611,7 @@ static void __ufshcd_release(struct ufs_hba *hba) return; hba->clk_gating.state = REQ_CLKS_OFF; + trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state); schedule_delayed_work(&hba->clk_gating.gate_work, msecs_to_jiffies(hba->clk_gating.delay_ms)); } @@ -881,6 +1649,41 @@ static ssize_t ufshcd_clkgate_delay_store(struct device *dev, return count; } +static ssize_t ufshcd_clkgate_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled); +} + +static ssize_t ufshcd_clkgate_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + unsigned long flags; + u32 value; + + if (kstrtou32(buf, 0, &value)) + return -EINVAL; + + value = !!value; + if (value == hba->clk_gating.is_enabled) + goto out; + + if (value) { + ufshcd_release(hba); + } else { + spin_lock_irqsave(hba->host->host_lock, flags); + hba->clk_gating.active_reqs++; + spin_unlock_irqrestore(hba->host->host_lock, flags); + } + + hba->clk_gating.is_enabled = value; +out: + return count; +} + static void ufshcd_init_clk_gating(struct ufs_hba *hba) { if (!ufshcd_is_clkgating_allowed(hba)) @@ -890,13 +1693,23 @@ static void ufshcd_init_clk_gating(struct ufs_hba *hba) INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); + hba->clk_gating.is_enabled = true; + hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show; hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store; sysfs_attr_init(&hba->clk_gating.delay_attr.attr); hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms"; - hba->clk_gating.delay_attr.attr.mode = S_IRUGO | S_IWUSR; + hba->clk_gating.delay_attr.attr.mode = 0644; if (device_create_file(hba->dev, &hba->clk_gating.delay_attr)) dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n"); + + hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show; + hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store; + sysfs_attr_init(&hba->clk_gating.enable_attr.attr); + hba->clk_gating.enable_attr.attr.name = "clkgate_enable"; + hba->clk_gating.enable_attr.attr.mode = 0644; + if (device_create_file(hba->dev, &hba->clk_gating.enable_attr)) + dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n"); } static void ufshcd_exit_clk_gating(struct ufs_hba *hba) @@ -904,6 +1717,7 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) if (!ufshcd_is_clkgating_allowed(hba)) return; device_remove_file(hba->dev, &hba->clk_gating.delay_attr); + device_remove_file(hba->dev, &hba->clk_gating.enable_attr); cancel_work_sync(&hba->clk_gating.ungate_work); cancel_delayed_work_sync(&hba->clk_gating.gate_work); } @@ -911,9 +1725,27 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) /* Must be called with host lock acquired */ static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) { - if (!ufshcd_is_clkscaling_enabled(hba)) + bool queue_resume_work = false; + + if (!ufshcd_is_clkscaling_supported(hba)) + return; + + if (!hba->clk_scaling.active_reqs++) + queue_resume_work = true; + + if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress) return; + if (queue_resume_work) + queue_work(hba->clk_scaling.workq, + &hba->clk_scaling.resume_work); + + if (!hba->clk_scaling.window_start_t) { + hba->clk_scaling.window_start_t = jiffies; + hba->clk_scaling.tot_busy_t = 0; + hba->clk_scaling.is_busy_started = false; + } + if (!hba->clk_scaling.is_busy_started) { hba->clk_scaling.busy_start_t = ktime_get(); hba->clk_scaling.is_busy_started = true; @@ -924,7 +1756,7 @@ static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) { struct ufs_clk_scaling *scaling = &hba->clk_scaling; - if (!ufshcd_is_clkscaling_enabled(hba)) + if (!ufshcd_is_clkscaling_supported(hba)) return; if (!hba->outstanding_reqs && scaling->is_busy_started) { @@ -942,11 +1774,13 @@ static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) static inline void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) { + hba->lrb[task_tag].issue_time_stamp = ktime_get(); ufshcd_clk_scaling_start_busy(hba); __set_bit(task_tag, &hba->outstanding_reqs); ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); /* Make sure that doorbell is committed immediately */ wmb(); + ufshcd_add_command_trace(hba, task_tag, "send"); } /** @@ -1484,6 +2318,9 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) BUG(); } + if (!down_read_trylock(&hba->clk_scaling_lock)) + return SCSI_MLQUEUE_HOST_BUSY; + spin_lock_irqsave(hba->host->host_lock, flags); switch (hba->ufshcd_state) { case UFSHCD_STATE_OPERATIONAL: @@ -1512,6 +2349,8 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) } spin_unlock_irqrestore(hba->host->host_lock, flags); + hba->req_abort_count = 0; + /* acquire the tag to make sure device cmds don't use it */ if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) { /* @@ -1541,6 +2380,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) lrbp->task_tag = tag; lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false; + lrbp->req_abort_skip = false; ufshcd_comp_scsi_upiu(hba, lrbp); @@ -1560,6 +2400,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) out_unlock: spin_unlock_irqrestore(hba->host->host_lock, flags); out: + up_read(&hba->clk_scaling_lock); return err; } @@ -1622,6 +2463,7 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) int resp; int err = 0; + hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); switch (resp) { @@ -1748,6 +2590,8 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, struct completion wait; unsigned long flags; + down_read(&hba->clk_scaling_lock); + /* * Get free slot, sleep if slots are unavailable. * Even though we use wait_event() which sleeps indefinitely, @@ -1776,6 +2620,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, out_put_tag: ufshcd_put_dev_cmd_tag(hba, tag); wake_up(&hba->dev_cmd.tag_wq); + up_read(&hba->clk_scaling_lock); return err; } @@ -2073,9 +2918,11 @@ out: * The buf_len parameter will contain, on return, the length parameter * received on the response. */ -int ufshcd_query_descriptor_retry(struct ufs_hba *hba, - enum query_opcode opcode, enum desc_idn idn, u8 index, - u8 selector, u8 *desc_buf, int *buf_len) +static int ufshcd_query_descriptor_retry(struct ufs_hba *hba, + enum query_opcode opcode, + enum desc_idn idn, u8 index, + u8 selector, + u8 *desc_buf, int *buf_len) { int err; int retries; @@ -2089,7 +2936,6 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, return err; } -EXPORT_SYMBOL(ufshcd_query_descriptor_retry); /** * ufshcd_read_desc_param - read the specified descriptor parameter @@ -2207,11 +3053,10 @@ static inline int ufshcd_read_power_desc(struct ufs_hba *hba, return err; } -int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size) +static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size) { return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size); } -EXPORT_SYMBOL(ufshcd_read_device_desc); /** * ufshcd_read_string_desc - read string descriptor @@ -2223,8 +3068,9 @@ EXPORT_SYMBOL(ufshcd_read_device_desc); * * Return 0 in case of success, non-zero otherwise */ -int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index, u8 *buf, - u32 size, bool ascii) +#define ASCII_STD true +static int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index, + u8 *buf, u32 size, bool ascii) { int err = 0; @@ -2280,7 +3126,6 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index, u8 *buf, out: return err; } -EXPORT_SYMBOL(ufshcd_read_string_desc); /** * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter @@ -2453,12 +3298,19 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) } hba->lrb[i].utr_descriptor_ptr = (utrdlp + i); + hba->lrb[i].utrd_dma_addr = hba->utrdl_dma_addr + + (i * sizeof(struct utp_transfer_req_desc)); hba->lrb[i].ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i); + hba->lrb[i].ucd_req_dma_addr = cmd_desc_element_addr; hba->lrb[i].ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu; + hba->lrb[i].ucd_rsp_dma_addr = cmd_desc_element_addr + + response_offset; hba->lrb[i].ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table; + hba->lrb[i].ucd_prdt_dma_addr = cmd_desc_element_addr + + prdt_offset; } } @@ -2482,7 +3334,7 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba) ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) - dev_err(hba->dev, + dev_dbg(hba->dev, "dme-link-startup: error code %d\n", ret); return ret; } @@ -2702,6 +3554,12 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) ret = (status != PWR_OK) ? status : -1; } out: + if (ret) { + ufshcd_print_host_state(hba); + ufshcd_print_pwr_info(hba); + ufshcd_print_host_regs(hba); + } + spin_lock_irqsave(hba->host->host_lock, flags); hba->active_uic_cmd = NULL; hba->uic_async_done = NULL; @@ -2776,11 +3634,14 @@ static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba) { int ret; struct uic_command uic_cmd = {0}; + ktime_t start = ktime_get(); ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); uic_cmd.command = UIC_CMD_DME_HIBER_ENTER; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); + trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter", + ktime_to_us(ktime_sub(ktime_get(), start)), ret); if (ret) { dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n", @@ -2816,18 +3677,25 @@ static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) { struct uic_command uic_cmd = {0}; int ret; + ktime_t start = ktime_get(); ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); uic_cmd.command = UIC_CMD_DME_HIBER_EXIT; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); + trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit", + ktime_to_us(ktime_sub(ktime_get(), start)), ret); + if (ret) { dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n", __func__, ret); ret = ufshcd_link_recovery(hba); - } else + } else { ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, POST_CHANGE); + hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get(); + hba->ufs_stats.hibern8_exit_cnt++; + } return ret; } @@ -2994,6 +3862,8 @@ static int ufshcd_config_pwr_mode(struct ufs_hba *hba, memcpy(&final_params, desired_pwr_mode, sizeof(final_params)); ret = ufshcd_change_power_mode(hba, &final_params); + if (!ret) + ufshcd_print_pwr_info(hba); return ret; } @@ -3265,6 +4135,10 @@ link_startup: goto link_startup; } + /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ + ufshcd_init_pwr_info(hba); + ufshcd_print_pwr_info(hba); + if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { ret = ufshcd_disable_device_tx_lcc(hba); if (ret) @@ -3278,8 +4152,12 @@ link_startup: ret = ufshcd_make_hba_operational(hba); out: - if (ret) + if (ret) { dev_err(hba->dev, "link startup failed %d\n", ret); + ufshcd_print_host_state(hba); + ufshcd_print_pwr_info(hba); + ufshcd_print_host_regs(hba); + } return ret; } @@ -3591,7 +4469,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) switch (ocs) { case OCS_SUCCESS: result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); - + hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); switch (result) { case UPIU_TRANSACTION_RESPONSE: /* @@ -3652,10 +4530,15 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) default: result |= DID_ERROR << 16; dev_err(hba->dev, - "OCS error from controller = %x\n", ocs); + "OCS error from controller = %x for tag %d\n", + ocs, lrbp->task_tag); + ufshcd_print_host_regs(hba); + ufshcd_print_host_state(hba); break; } /* end of switch */ + if (host_byte(result) != DID_OK) + ufshcd_print_trs(hba, 1 << lrbp->task_tag, true); return result; } @@ -3695,6 +4578,7 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, lrbp = &hba->lrb[index]; cmd = lrbp->cmd; if (cmd) { + ufshcd_add_command_trace(hba, index, "complete"); result = ufshcd_transfer_rsp_status(hba, lrbp); scsi_dma_unmap(cmd); cmd->result = result; @@ -3706,9 +4590,16 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, __ufshcd_release(hba); } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE || lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) { - if (hba->dev_cmd.complete) + if (hba->dev_cmd.complete) { + ufshcd_add_command_trace(hba, index, + "dev_complete"); complete(hba->dev_cmd.complete); + } } + if (ufshcd_is_clkscaling_supported(hba)) + hba->clk_scaling.active_reqs--; + if (ufshcd_is_clkscaling_supported(hba)) + hba->clk_scaling.active_reqs--; } /* clear corresponding bits of completed commands */ @@ -3828,6 +4719,7 @@ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = true; + trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled"); /* No need of URGENT_BKOPS exception from the device */ err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); @@ -3878,23 +4770,31 @@ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = false; + trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled"); out: return err; } /** - * ufshcd_force_reset_auto_bkops - force enable of auto bkops + * ufshcd_force_reset_auto_bkops - force reset auto bkops state * @hba: per adapter instance * * After a device reset the device may toggle the BKOPS_EN flag * to default value. The s/w tracking variables should be updated - * as well. Do this by forcing enable of auto bkops. + * as well. This function would change the auto-bkops state based on + * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND. */ -static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) +static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) { - hba->auto_bkops_enabled = false; - hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; - ufshcd_enable_auto_bkops(hba); + if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) { + hba->auto_bkops_enabled = false; + hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS; + ufshcd_enable_auto_bkops(hba); + } else { + hba->auto_bkops_enabled = true; + hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS; + ufshcd_disable_auto_bkops(hba); + } } static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) @@ -4246,6 +5146,14 @@ out: pm_runtime_put_sync(hba->dev); } +static void ufshcd_update_uic_reg_hist(struct ufs_uic_err_reg_hist *reg_hist, + u32 reg) +{ + reg_hist->reg[reg_hist->pos] = reg; + reg_hist->tstamp[reg_hist->pos] = ktime_get(); + reg_hist->pos = (reg_hist->pos + 1) % UIC_ERR_REG_HIST_LENGTH; +} + /** * ufshcd_update_uic_error - check and set fatal UIC error flags. * @hba: per-adapter instance @@ -4258,15 +5166,20 @@ static void ufshcd_update_uic_error(struct ufs_hba *hba) reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); /* Ignore LINERESET indication, as this is not an error */ if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) && - (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) + (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) { /* * To know whether this error is fatal or not, DB timeout * must be checked but this error is handled separately. */ dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__); + ufshcd_update_uic_reg_hist(&hba->ufs_stats.pa_err, reg); + } /* PA_INIT_ERROR is fatal and needs UIC reset */ reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER); + if (reg) + ufshcd_update_uic_reg_hist(&hba->ufs_stats.dl_err, reg); + if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT) hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR; else if (hba->dev_quirks & @@ -4280,16 +5193,22 @@ static void ufshcd_update_uic_error(struct ufs_hba *hba) /* UIC NL/TL/DME errors needs software retry */ reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER); - if (reg) + if (reg) { + ufshcd_update_uic_reg_hist(&hba->ufs_stats.nl_err, reg); hba->uic_error |= UFSHCD_UIC_NL_ERROR; + } reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER); - if (reg) + if (reg) { + ufshcd_update_uic_reg_hist(&hba->ufs_stats.tl_err, reg); hba->uic_error |= UFSHCD_UIC_TL_ERROR; + } reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME); - if (reg) + if (reg) { + ufshcd_update_uic_reg_hist(&hba->ufs_stats.dme_err, reg); hba->uic_error |= UFSHCD_UIC_DME_ERROR; + } dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n", __func__, hba->uic_error); @@ -4327,6 +5246,22 @@ static void ufshcd_check_errors(struct ufs_hba *hba) scsi_block_requests(hba->host); hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED; + + /* dump controller state before resetting */ + if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) { + bool pr_prdt = !!(hba->saved_err & + SYSTEM_BUS_FATAL_ERROR); + + dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n", + __func__, hba->saved_err, + hba->saved_uic_err); + + ufshcd_print_host_regs(hba); + ufshcd_print_pwr_info(hba); + ufshcd_print_tmrs(hba, hba->outstanding_tasks); + ufshcd_print_trs(hba, hba->outstanding_reqs, + pr_prdt); + } schedule_work(&hba->eh_work); } } @@ -4557,7 +5492,9 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) spin_lock_irqsave(host->host_lock, flags); ufshcd_transfer_req_compl(hba); spin_unlock_irqrestore(host->host_lock, flags); + out: + hba->req_abort_count = 0; if (!err) { err = SUCCESS; } else { @@ -4567,6 +5504,17 @@ out: return err; } +static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) +{ + struct ufshcd_lrb *lrbp; + int tag; + + for_each_set_bit(tag, &bitmap, hba->nutrs) { + lrbp = &hba->lrb[tag]; + lrbp->req_abort_skip = true; + } +} + /** * ufshcd_abort - abort a specific command * @cmd: SCSI command pointer @@ -4594,6 +5542,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) host = cmd->device->host; hba = shost_priv(host); tag = cmd->request->tag; + lrbp = &hba->lrb[tag]; if (!ufshcd_valid_tag(hba, tag)) { dev_err(hba->dev, "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p", @@ -4601,6 +5550,16 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) BUG(); } + /* + * Task abort to the device W-LUN is illegal. When this command + * will fail, due to spec violation, scsi err handling next step + * will be to send LU reset which, again, is a spec violation. + * To avoid these unnecessary/illegal step we skip to the last error + * handling stage: reset and restore. + */ + if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) + return ufshcd_eh_host_reset_handler(cmd); + ufshcd_hold(hba, false); reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); /* If command is already aborted/completed, return SUCCESS */ @@ -4617,18 +5576,48 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) __func__, tag); } - lrbp = &hba->lrb[tag]; + /* Print Transfer Request of aborted task */ + dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag); + + /* + * Print detailed info about aborted request. + * As more than one request might get aborted at the same time, + * print full information only for the first aborted request in order + * to reduce repeated printouts. For other aborted requests only print + * basic details. + */ + scsi_print_command(hba->lrb[tag].cmd); + if (!hba->req_abort_count) { + ufshcd_print_host_regs(hba); + ufshcd_print_host_state(hba); + ufshcd_print_pwr_info(hba); + ufshcd_print_trs(hba, 1 << tag, true); + } else { + ufshcd_print_trs(hba, 1 << tag, false); + } + hba->req_abort_count++; + + /* Skip task abort in case previous aborts failed and report failure */ + if (lrbp->req_abort_skip) { + err = -EIO; + goto out; + } + for (poll_cnt = 100; poll_cnt; poll_cnt--) { err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, UFS_QUERY_TASK, &resp); if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { /* cmd pending in the device */ + dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", + __func__, tag); break; } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) { /* * cmd not pending in the device, check if it is * in transition. */ + dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n", + __func__, tag); reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); if (reg & (1 << tag)) { /* sleep for max. 200us to stabilize */ @@ -4636,8 +5625,13 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) continue; } /* command completed already */ + dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n", + __func__, tag); goto out; } else { + dev_err(hba->dev, + "%s: no response from device. tag = %d, err %d\n", + __func__, tag, err); if (!err) err = resp; /* service response error */ goto out; @@ -4652,14 +5646,20 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, UFS_ABORT_TASK, &resp); if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { - if (!err) + if (!err) { err = resp; /* service response error */ + dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", + __func__, tag, err); + } goto out; } err = ufshcd_clear_cmd(hba, tag); - if (err) + if (err) { + dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", + __func__, tag, err); goto out; + } scsi_dma_unmap(cmd); @@ -4676,6 +5676,7 @@ out: err = SUCCESS; } else { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); + ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); err = FAILED; } @@ -4707,6 +5708,9 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) ufshcd_hba_stop(hba, false); spin_unlock_irqrestore(hba->host->host_lock, flags); + /* scale up clocks to max frequency before full reinitialization */ + ufshcd_scale_clks(hba, true); + err = ufshcd_hba_enable(hba); if (err) goto out; @@ -4822,7 +5826,7 @@ static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff) u16 unit; for (i = start_scan; i >= 0; i--) { - data = be16_to_cpu(*((u16 *)(buff + 2*i))); + data = be16_to_cpup((__be16 *)&buff[2 * i]); unit = (data & ATTR_ICC_LVL_UNIT_MASK) >> ATTR_ICC_LVL_UNIT_OFFSET; curr_uA = data & ATTR_ICC_LVL_VALUE_MASK; @@ -5008,8 +6012,8 @@ out: return ret; } -static int ufs_get_device_info(struct ufs_hba *hba, - struct ufs_device_info *card_data) +static int ufs_get_device_desc(struct ufs_hba *hba, + struct ufs_dev_desc *dev_desc) { int err; u8 model_index; @@ -5028,7 +6032,7 @@ static int ufs_get_device_info(struct ufs_hba *hba, * getting vendor (manufacturerID) and Bank Index in big endian * format */ - card_data->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | + dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1]; model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; @@ -5042,36 +6046,26 @@ static int ufs_get_device_info(struct ufs_hba *hba, } str_desc_buf[QUERY_DESC_STRING_MAX_SIZE] = '\0'; - strlcpy(card_data->model, (str_desc_buf + QUERY_DESC_HDR_SIZE), + strlcpy(dev_desc->model, (str_desc_buf + QUERY_DESC_HDR_SIZE), min_t(u8, str_desc_buf[QUERY_DESC_LENGTH_OFFSET], MAX_MODEL_LEN)); /* Null terminate the model string */ - card_data->model[MAX_MODEL_LEN] = '\0'; + dev_desc->model[MAX_MODEL_LEN] = '\0'; out: return err; } -void ufs_advertise_fixup_device(struct ufs_hba *hba) +static void ufs_fixup_device_setup(struct ufs_hba *hba, + struct ufs_dev_desc *dev_desc) { - int err; struct ufs_dev_fix *f; - struct ufs_device_info card_data; - - card_data.wmanufacturerid = 0; - - err = ufs_get_device_info(hba, &card_data); - if (err) { - dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", - __func__, err); - return; - } for (f = ufs_fixups; f->quirk; f++) { - if (((f->card.wmanufacturerid == card_data.wmanufacturerid) || - (f->card.wmanufacturerid == UFS_ANY_VENDOR)) && - (STR_PRFX_EQUAL(f->card.model, card_data.model) || + if ((f->card.wmanufacturerid == dev_desc->wmanufacturerid || + f->card.wmanufacturerid == UFS_ANY_VENDOR) && + (STR_PRFX_EQUAL(f->card.model, dev_desc->model) || !strcmp(f->card.model, UFS_ANY_MODEL))) hba->dev_quirks |= f->quirk; } @@ -5241,6 +6235,22 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) ufshcd_vops_apply_dev_quirks(hba); } +static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) +{ + int err_reg_hist_size = sizeof(struct ufs_uic_err_reg_hist); + + hba->ufs_stats.hibern8_exit_cnt = 0; + hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); + + memset(&hba->ufs_stats.pa_err, 0, err_reg_hist_size); + memset(&hba->ufs_stats.dl_err, 0, err_reg_hist_size); + memset(&hba->ufs_stats.nl_err, 0, err_reg_hist_size); + memset(&hba->ufs_stats.tl_err, 0, err_reg_hist_size); + memset(&hba->ufs_stats.dme_err, 0, err_reg_hist_size); + + hba->req_abort_count = 0; +} + /** * ufshcd_probe_hba - probe hba to detect device and initialize * @hba: per-adapter instance @@ -5249,18 +6259,21 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) */ static int ufshcd_probe_hba(struct ufs_hba *hba) { + struct ufs_dev_desc card = {0}; int ret; + ktime_t start = ktime_get(); ret = ufshcd_link_startup(hba); if (ret) goto out; - ufshcd_init_pwr_info(hba); - /* set the default level for urgent bkops */ hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; hba->is_urgent_bkops_lvl_checked = false; + /* Debug counters initialization */ + ufshcd_clear_dbg_ufs_stats(hba); + /* UniPro link is active now */ ufshcd_set_link_active(hba); @@ -5272,7 +6285,14 @@ static int ufshcd_probe_hba(struct ufs_hba *hba) if (ret) goto out; - ufs_advertise_fixup_device(hba); + ret = ufs_get_device_desc(hba, &card); + if (ret) { + dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", + __func__, ret); + goto out; + } + + ufs_fixup_device_setup(hba, &card); ufshcd_tune_unipro_params(hba); ret = ufshcd_set_vccq_rail_unused(hba, @@ -5320,6 +6340,27 @@ static int ufshcd_probe_hba(struct ufs_hba *hba) if (ufshcd_scsi_add_wlus(hba)) goto out; + /* Initialize devfreq after UFS device is detected */ + if (ufshcd_is_clkscaling_supported(hba)) { + memcpy(&hba->clk_scaling.saved_pwr_info.info, + &hba->pwr_info, + sizeof(struct ufs_pa_layer_attr)); + hba->clk_scaling.saved_pwr_info.is_valid = true; + if (!hba->devfreq) { + hba->devfreq = devm_devfreq_add_device(hba->dev, + &ufs_devfreq_profile, + "simple_ondemand", + NULL); + if (IS_ERR(hba->devfreq)) { + ret = PTR_ERR(hba->devfreq); + dev_err(hba->dev, "Unable to register with devfreq %d\n", + ret); + goto out; + } + } + hba->clk_scaling.is_allowed = true; + } + scsi_scan_host(hba->host); pm_runtime_put_sync(hba->dev); } @@ -5327,9 +6368,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba) if (!hba->is_init_prefetch) hba->is_init_prefetch = true; - /* Resume devfreq after UFS device is detected */ - ufshcd_resume_clkscaling(hba); - out: /* * If we failed to initialize the device or the device is not @@ -5340,6 +6378,9 @@ out: ufshcd_hba_exit(hba); } + trace_ufshcd_init(dev_name(hba->dev), ret, + ktime_to_us(ktime_sub(ktime_get(), start)), + hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; } @@ -5650,6 +6691,8 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; unsigned long flags; + ktime_t start = ktime_get(); + bool clk_state_changed = false; if (!head || list_empty(head)) goto out; @@ -5663,6 +6706,7 @@ static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on, if (skip_ref_clk && !strcmp(clki->name, "ref_clk")) continue; + clk_state_changed = on ^ clki->enabled; if (on && !clki->enabled) { ret = clk_prepare_enable(clki->clk); if (ret) { @@ -5689,11 +6733,18 @@ out: if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) clk_disable_unprepare(clki->clk); } - } else if (on) { + } else if (!ret && on) { spin_lock_irqsave(hba->host->host_lock, flags); hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(dev_name(hba->dev), + hba->clk_gating.state); spin_unlock_irqrestore(hba->host->host_lock, flags); } + + if (clk_state_changed) + trace_ufshcd_profile_clk_gating(dev_name(hba->dev), + (on ? "on" : "off"), + ktime_to_us(ktime_sub(ktime_get(), start)), ret); return ret; } @@ -5835,6 +6886,11 @@ static void ufshcd_hba_exit(struct ufs_hba *hba) ufshcd_variant_hba_exit(hba); ufshcd_setup_vreg(hba, false); ufshcd_suspend_clkscaling(hba); + if (ufshcd_is_clkscaling_supported(hba)) { + if (hba->devfreq) + ufshcd_suspend_clkscaling(hba); + destroy_workqueue(hba->clk_scaling.workq); + } ufshcd_setup_clocks(hba, false); ufshcd_setup_hba_vreg(hba, false); hba->is_powered = false; @@ -5859,9 +6915,9 @@ ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp) goto out; } - ret = scsi_execute_req_flags(sdp, cmd, DMA_FROM_DEVICE, buffer, - UFSHCD_REQ_SENSE_SIZE, NULL, - msecs_to_jiffies(1000), 3, NULL, 0, RQF_PM); + ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer, + UFSHCD_REQ_SENSE_SIZE, NULL, NULL, + msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL); if (ret) pr_err("%s: failed with err %d\n", __func__, ret); @@ -5926,8 +6982,8 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, * callbacks hence set the RQF_PM flag so that it doesn't resume the * already suspended childs. */ - ret = scsi_execute_req_flags(sdp, cmd, DMA_NONE, NULL, 0, &sshdr, - START_STOP_TIMEOUT, 0, NULL, 0, RQF_PM); + ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr, + START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL); if (ret) { sdev_printk(KERN_WARNING, sdp, "START_STOP failed for power mode: %d, result %x\n", @@ -6110,7 +7166,11 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) ufshcd_hold(hba, false); hba->clk_gating.is_suspended = true; - ufshcd_suspend_clkscaling(hba); + if (hba->clk_scaling.is_allowed) { + cancel_work_sync(&hba->clk_scaling.suspend_work); + cancel_work_sync(&hba->clk_scaling.resume_work); + ufshcd_suspend_clkscaling(hba); + } if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE && req_link_state == UIC_LINK_ACTIVE_STATE) { @@ -6176,6 +7236,7 @@ disable_clks: __ufshcd_setup_clocks(hba, false, true); hba->clk_gating.state = CLKS_OFF; + trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state); /* * Disable the host irq as host controller as there won't be any * host controller transaction expected till resume. @@ -6186,7 +7247,8 @@ disable_clks: goto out; set_link_active: - ufshcd_resume_clkscaling(hba); + if (hba->clk_scaling.is_allowed) + ufshcd_resume_clkscaling(hba); ufshcd_vreg_set_hpm(hba); if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba)) ufshcd_set_link_active(hba); @@ -6196,7 +7258,8 @@ set_dev_active: if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE)) ufshcd_disable_auto_bkops(hba); enable_gating: - ufshcd_resume_clkscaling(hba); + if (hba->clk_scaling.is_allowed) + ufshcd_resume_clkscaling(hba); hba->clk_gating.is_suspended = false; ufshcd_release(hba); out: @@ -6268,14 +7331,19 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) goto set_old_link_state; } - /* - * If BKOPs operations are urgently needed at this moment then - * keep auto-bkops enabled or else disable it. - */ - ufshcd_urgent_bkops(hba); + if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) + ufshcd_enable_auto_bkops(hba); + else + /* + * If BKOPs operations are urgently needed at this moment then + * keep auto-bkops enabled or else disable it. + */ + ufshcd_urgent_bkops(hba); + hba->clk_gating.is_suspended = false; - ufshcd_resume_clkscaling(hba); + if (hba->clk_scaling.is_allowed) + ufshcd_resume_clkscaling(hba); /* Schedule clock gating in case of no access to UFS device yet */ ufshcd_release(hba); @@ -6289,7 +7357,8 @@ disable_vreg: ufshcd_vreg_set_lpm(hba); disable_irq_and_vops_clks: ufshcd_disable_irq(hba); - ufshcd_suspend_clkscaling(hba); + if (hba->clk_scaling.is_allowed) + ufshcd_suspend_clkscaling(hba); ufshcd_setup_clocks(hba, false); out: hba->pm_op_in_progress = 0; @@ -6308,6 +7377,7 @@ out: int ufshcd_system_suspend(struct ufs_hba *hba) { int ret = 0; + ktime_t start = ktime_get(); if (!hba || !hba->is_powered) return 0; @@ -6334,6 +7404,9 @@ int ufshcd_system_suspend(struct ufs_hba *hba) ret = ufshcd_suspend(hba, UFS_SYSTEM_PM); out: + trace_ufshcd_system_suspend(dev_name(hba->dev), ret, + ktime_to_us(ktime_sub(ktime_get(), start)), + hba->curr_dev_pwr_mode, hba->uic_link_state); if (!ret) hba->is_sys_suspended = true; return ret; @@ -6349,6 +7422,9 @@ EXPORT_SYMBOL(ufshcd_system_suspend); int ufshcd_system_resume(struct ufs_hba *hba) { + int ret = 0; + ktime_t start = ktime_get(); + if (!hba) return -EINVAL; @@ -6357,9 +7433,14 @@ int ufshcd_system_resume(struct ufs_hba *hba) * Let the runtime resume take care of resuming * if runtime suspended. */ - return 0; - - return ufshcd_resume(hba, UFS_SYSTEM_PM); + goto out; + else + ret = ufshcd_resume(hba, UFS_SYSTEM_PM); +out: + trace_ufshcd_system_resume(dev_name(hba->dev), ret, + ktime_to_us(ktime_sub(ktime_get(), start)), + hba->curr_dev_pwr_mode, hba->uic_link_state); + return ret; } EXPORT_SYMBOL(ufshcd_system_resume); @@ -6373,13 +7454,21 @@ EXPORT_SYMBOL(ufshcd_system_resume); */ int ufshcd_runtime_suspend(struct ufs_hba *hba) { + int ret = 0; + ktime_t start = ktime_get(); + if (!hba) return -EINVAL; if (!hba->is_powered) - return 0; - - return ufshcd_suspend(hba, UFS_RUNTIME_PM); + goto out; + else + ret = ufshcd_suspend(hba, UFS_RUNTIME_PM); +out: + trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret, + ktime_to_us(ktime_sub(ktime_get(), start)), + hba->curr_dev_pwr_mode, hba->uic_link_state); + return ret; } EXPORT_SYMBOL(ufshcd_runtime_suspend); @@ -6406,13 +7495,21 @@ EXPORT_SYMBOL(ufshcd_runtime_suspend); */ int ufshcd_runtime_resume(struct ufs_hba *hba) { + int ret = 0; + ktime_t start = ktime_get(); + if (!hba) return -EINVAL; if (!hba->is_powered) - return 0; - - return ufshcd_resume(hba, UFS_RUNTIME_PM); + goto out; + else + ret = ufshcd_resume(hba, UFS_RUNTIME_PM); +out: + trace_ufshcd_runtime_resume(dev_name(hba->dev), ret, + ktime_to_us(ktime_sub(ktime_get(), start)), + hba->curr_dev_pwr_mode, hba->uic_link_state); + return ret; } EXPORT_SYMBOL(ufshcd_runtime_resume); @@ -6422,6 +7519,127 @@ int ufshcd_runtime_idle(struct ufs_hba *hba) } EXPORT_SYMBOL(ufshcd_runtime_idle); +static inline ssize_t ufshcd_pm_lvl_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count, + bool rpm) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + unsigned long flags, value; + + if (kstrtoul(buf, 0, &value)) + return -EINVAL; + + if ((value < UFS_PM_LVL_0) || (value >= UFS_PM_LVL_MAX)) + return -EINVAL; + + spin_lock_irqsave(hba->host->host_lock, flags); + if (rpm) + hba->rpm_lvl = value; + else + hba->spm_lvl = value; + spin_unlock_irqrestore(hba->host->host_lock, flags); + return count; +} + +static ssize_t ufshcd_rpm_lvl_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + int curr_len; + u8 lvl; + + curr_len = snprintf(buf, PAGE_SIZE, + "\nCurrent Runtime PM level [%d] => dev_state [%s] link_state [%s]\n", + hba->rpm_lvl, + ufschd_ufs_dev_pwr_mode_to_string( + ufs_pm_lvl_states[hba->rpm_lvl].dev_state), + ufschd_uic_link_state_to_string( + ufs_pm_lvl_states[hba->rpm_lvl].link_state)); + + curr_len += snprintf((buf + curr_len), (PAGE_SIZE - curr_len), + "\nAll available Runtime PM levels info:\n"); + for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) + curr_len += snprintf((buf + curr_len), (PAGE_SIZE - curr_len), + "\tRuntime PM level [%d] => dev_state [%s] link_state [%s]\n", + lvl, + ufschd_ufs_dev_pwr_mode_to_string( + ufs_pm_lvl_states[lvl].dev_state), + ufschd_uic_link_state_to_string( + ufs_pm_lvl_states[lvl].link_state)); + + return curr_len; +} + +static ssize_t ufshcd_rpm_lvl_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + return ufshcd_pm_lvl_store(dev, attr, buf, count, true); +} + +static void ufshcd_add_rpm_lvl_sysfs_nodes(struct ufs_hba *hba) +{ + hba->rpm_lvl_attr.show = ufshcd_rpm_lvl_show; + hba->rpm_lvl_attr.store = ufshcd_rpm_lvl_store; + sysfs_attr_init(&hba->rpm_lvl_attr.attr); + hba->rpm_lvl_attr.attr.name = "rpm_lvl"; + hba->rpm_lvl_attr.attr.mode = 0644; + if (device_create_file(hba->dev, &hba->rpm_lvl_attr)) + dev_err(hba->dev, "Failed to create sysfs for rpm_lvl\n"); +} + +static ssize_t ufshcd_spm_lvl_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + int curr_len; + u8 lvl; + + curr_len = snprintf(buf, PAGE_SIZE, + "\nCurrent System PM level [%d] => dev_state [%s] link_state [%s]\n", + hba->spm_lvl, + ufschd_ufs_dev_pwr_mode_to_string( + ufs_pm_lvl_states[hba->spm_lvl].dev_state), + ufschd_uic_link_state_to_string( + ufs_pm_lvl_states[hba->spm_lvl].link_state)); + + curr_len += snprintf((buf + curr_len), (PAGE_SIZE - curr_len), + "\nAll available System PM levels info:\n"); + for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) + curr_len += snprintf((buf + curr_len), (PAGE_SIZE - curr_len), + "\tSystem PM level [%d] => dev_state [%s] link_state [%s]\n", + lvl, + ufschd_ufs_dev_pwr_mode_to_string( + ufs_pm_lvl_states[lvl].dev_state), + ufschd_uic_link_state_to_string( + ufs_pm_lvl_states[lvl].link_state)); + + return curr_len; +} + +static ssize_t ufshcd_spm_lvl_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + return ufshcd_pm_lvl_store(dev, attr, buf, count, false); +} + +static void ufshcd_add_spm_lvl_sysfs_nodes(struct ufs_hba *hba) +{ + hba->spm_lvl_attr.show = ufshcd_spm_lvl_show; + hba->spm_lvl_attr.store = ufshcd_spm_lvl_store; + sysfs_attr_init(&hba->spm_lvl_attr.attr); + hba->spm_lvl_attr.attr.name = "spm_lvl"; + hba->spm_lvl_attr.attr.mode = 0644; + if (device_create_file(hba->dev, &hba->spm_lvl_attr)) + dev_err(hba->dev, "Failed to create sysfs for spm_lvl\n"); +} + +static inline void ufshcd_add_sysfs_nodes(struct ufs_hba *hba) +{ + ufshcd_add_rpm_lvl_sysfs_nodes(hba); + ufshcd_add_spm_lvl_sysfs_nodes(hba); +} + /** * ufshcd_shutdown - shutdown routine * @hba: per adapter instance @@ -6465,6 +7683,8 @@ void ufshcd_remove(struct ufs_hba *hba) ufshcd_hba_stop(hba, true); ufshcd_exit_clk_gating(hba); + if (ufshcd_is_clkscaling_supported(hba)) + device_remove_file(hba->dev, &hba->clk_scaling.enable_attr); ufshcd_hba_exit(hba); } EXPORT_SYMBOL_GPL(ufshcd_remove); @@ -6531,149 +7751,6 @@ out_error: } EXPORT_SYMBOL(ufshcd_alloc_host); -static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) -{ - int ret = 0; - struct ufs_clk_info *clki; - struct list_head *head = &hba->clk_list_head; - - if (!head || list_empty(head)) - goto out; - - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE); - if (ret) - return ret; - - list_for_each_entry(clki, head, list) { - if (!IS_ERR_OR_NULL(clki->clk)) { - if (scale_up && clki->max_freq) { - if (clki->curr_freq == clki->max_freq) - continue; - ret = clk_set_rate(clki->clk, clki->max_freq); - if (ret) { - dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", - __func__, clki->name, - clki->max_freq, ret); - break; - } - clki->curr_freq = clki->max_freq; - - } else if (!scale_up && clki->min_freq) { - if (clki->curr_freq == clki->min_freq) - continue; - ret = clk_set_rate(clki->clk, clki->min_freq); - if (ret) { - dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n", - __func__, clki->name, - clki->min_freq, ret); - break; - } - clki->curr_freq = clki->min_freq; - } - } - dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__, - clki->name, clk_get_rate(clki->clk)); - } - - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE); - -out: - return ret; -} - -static int ufshcd_devfreq_target(struct device *dev, - unsigned long *freq, u32 flags) -{ - int err = 0; - struct ufs_hba *hba = dev_get_drvdata(dev); - bool release_clk_hold = false; - unsigned long irq_flags; - - if (!ufshcd_is_clkscaling_enabled(hba)) - return -EINVAL; - - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (ufshcd_eh_in_progress(hba)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return 0; - } - - if (ufshcd_is_clkgating_allowed(hba) && - (hba->clk_gating.state != CLKS_ON)) { - if (cancel_delayed_work(&hba->clk_gating.gate_work)) { - /* hold the vote until the scaling work is completed */ - hba->clk_gating.active_reqs++; - release_clk_hold = true; - hba->clk_gating.state = CLKS_ON; - } else { - /* - * Clock gating work seems to be running in parallel - * hence skip scaling work to avoid deadlock between - * current scaling work and gating work. - */ - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return 0; - } - } - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - - if (*freq == UINT_MAX) - err = ufshcd_scale_clks(hba, true); - else if (*freq == 0) - err = ufshcd_scale_clks(hba, false); - - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (release_clk_hold) - __ufshcd_release(hba); - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - - return err; -} - -static int ufshcd_devfreq_get_dev_status(struct device *dev, - struct devfreq_dev_status *stat) -{ - struct ufs_hba *hba = dev_get_drvdata(dev); - struct ufs_clk_scaling *scaling = &hba->clk_scaling; - unsigned long flags; - - if (!ufshcd_is_clkscaling_enabled(hba)) - return -EINVAL; - - memset(stat, 0, sizeof(*stat)); - - spin_lock_irqsave(hba->host->host_lock, flags); - if (!scaling->window_start_t) - goto start_window; - - if (scaling->is_busy_started) - scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), - scaling->busy_start_t)); - - stat->total_time = jiffies_to_usecs((long)jiffies - - (long)scaling->window_start_t); - stat->busy_time = scaling->tot_busy_t; -start_window: - scaling->window_start_t = jiffies; - scaling->tot_busy_t = 0; - - if (hba->outstanding_reqs) { - scaling->busy_start_t = ktime_get(); - scaling->is_busy_started = true; - } else { - scaling->busy_start_t = 0; - scaling->is_busy_started = false; - } - spin_unlock_irqrestore(hba->host->host_lock, flags); - return 0; -} - -static struct devfreq_dev_profile ufs_devfreq_profile = { - .polling_ms = 100, - .target = ufshcd_devfreq_target, - .get_dev_status = ufshcd_devfreq_get_dev_status, -}; - /** * ufshcd_init - Driver initialization routine * @hba: per-adapter instance @@ -6757,6 +7834,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) /* Initialize mutex for device management commands */ mutex_init(&hba->dev_cmd.lock); + init_rwsem(&hba->clk_scaling_lock); + /* Initialize device management tag acquire wait queue */ init_waitqueue_head(&hba->dev_cmd.tag_wq); @@ -6795,22 +7874,38 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = ufshcd_hba_enable(hba); if (err) { dev_err(hba->dev, "Host controller enable failed\n"); + ufshcd_print_host_regs(hba); + ufshcd_print_host_state(hba); goto out_remove_scsi_host; } - if (ufshcd_is_clkscaling_enabled(hba)) { - hba->devfreq = devm_devfreq_add_device(dev, &ufs_devfreq_profile, - "simple_ondemand", NULL); - if (IS_ERR(hba->devfreq)) { - dev_err(hba->dev, "Unable to register with devfreq %ld\n", - PTR_ERR(hba->devfreq)); - err = PTR_ERR(hba->devfreq); - goto out_remove_scsi_host; - } - /* Suspend devfreq until the UFS device is detected */ - ufshcd_suspend_clkscaling(hba); + if (ufshcd_is_clkscaling_supported(hba)) { + char wq_name[sizeof("ufs_clkscaling_00")]; + + INIT_WORK(&hba->clk_scaling.suspend_work, + ufshcd_clk_scaling_suspend_work); + INIT_WORK(&hba->clk_scaling.resume_work, + ufshcd_clk_scaling_resume_work); + + snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clkscaling_%d", + host->host_no); + hba->clk_scaling.workq = create_singlethread_workqueue(wq_name); + + ufshcd_clkscaling_init_sysfs(hba); } + /* + * Set the default power management level for runtime and system PM. + * Default power saving mode is to keep UFS link in Hibern8 state + * and UFS device in sleep state. + */ + hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); @@ -6823,6 +7918,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_set_ufs_dev_active(hba); async_schedule(ufshcd_async_scan, hba); + ufshcd_add_sysfs_nodes(hba); return 0; |