scsi: megaraid_sas: Introduce various Aero performance modes

For Aero adapters, driver provides three different performance modes
controlled through module parameter named 'perf_mode'. Below are those
performance modes:

 0: Balanced - Additional high IOPS reply queues will be enabled along with
    low latency queues. Interrupt coalescing will be enabled only for these
    high IOPS reply queues.

 1: IOPS - No additional high IOPS queues are enabled. Interrupt coalescing
    will be enabled on all reply queues.

 2: Latency - No additional high IOPS queues are enabled. Interrupt
    coalescing will be disabled on all reply queues. This is a legacy
    behavior similar to Ventura & Invader Series.

Default performance mode settings:

 - Performance mode set to 'Balanced', if Aero controller is working in
   16GT/s PCIe speed.

 - Performance mode will be set to 'Latency' mode for all other cases.

Through module parameter 'perf_mode', user can override default performance
mode to desired one.

Captured some performance numbers with these performance modes.  4k Random
Read IO performance numbers on 24 SAS SSD drives for above three
performance modes. Performance data is from Intel Skylake and HGST SS300
(drive model SDLL1DLR400GCCA1).

IOPS:
 -----------------------------------------------------------------------
  |perf_mode    | qd = 1 | qd = 64 |   note                             |
  |-------------|--------|---------|-------------------------------------
  |balanced     |  259K  |  3061k  | Provides max performance numbers   |
  |             |        |         | both on lower QD workload &        |
  |             |        |         | also on higher QD workload         |
  |-------------|--------|---------|-------------------------------------
  |iops         |  220K  |  3100k  | Provides max performance numbers   |
  |             |        |         | only on higher QD workload.        |
  |-------------|--------|---------|-------------------------------------
  |latency      |  246k  |  2226k  | Provides good performance numbers  |
  |             |        |         | only on lower QD worklaod.         |
  -----------------------------------------------------------------------

Average Latency:
  -----------------------------------------------------
  |perf_mode    |  qd = 1      |    qd = 64           |
  |-------------|--------------|----------------------|
  |balanced     |  92.05 usec  |    501.12 usec       |
  |-------------|--------------|----------------------|
  |iops         |  108.40 usec |    498.10 usec       |
  |-------------|--------------|----------------------|
  |latency      |  97.10 usec  |    689.26 usec       |
  -----------------------------------------------------

Signed-off-by: Sumit Saxena <sumit.saxena@broadcom.com>
Signed-off-by: Chandrakanth Patil <chandrakanth.patil@broadcom.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

1 files changed, 15 insertions, 7 deletions

diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index 90dced4290a2..8a3e2551a6e1 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -1097,10 +1097,10 @@ megasas_ioc_init_fusion(struct megasas_instance *instance)
 
 	if ((instance->low_latency_index_start ==
 		MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing)
-		instance->balanced_mode = true;
+		instance->perf_mode = MR_BALANCED_PERF_MODE;
 
-	dev_info(&instance->pdev->dev, "Balanced mode :%s\n",
-		instance->balanced_mode ? "Yes" : "No");
+	dev_info(&instance->pdev->dev, "Performance mode :%s\n",
+		MEGASAS_PERF_MODE_2STR(instance->perf_mode));
 
 	instance->fw_sync_cache_support = (scratch_pad_1 &
 		MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
@@ -1190,9 +1190,17 @@ megasas_ioc_init_fusion(struct megasas_instance *instance)
 	 * Each bit in replyqueue_mask represents one group of MSI-x vectors
 	 * (each group has 8 vectors)
 	 */
-	if (instance->balanced_mode)
+	switch (instance->perf_mode) {
+	case MR_BALANCED_PERF_MODE:
 		init_frame->replyqueue_mask =
-		       cpu_to_le16(~(~0 << instance->low_latency_index_start / 8));
+		       cpu_to_le16(~(~0 << instance->low_latency_index_start/8));
+		break;
+	case MR_IOPS_PERF_MODE:
+		init_frame->replyqueue_mask =
+		       cpu_to_le16(~(~0 << instance->msix_vectors/8));
+		break;
+	}
+
 
 	req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
 	req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
@@ -2831,7 +2839,7 @@ megasas_build_ldio_fusion(struct megasas_instance *instance,
 			fp_possible = (io_info.fpOkForIo > 0) ? true : false;
 	}
 
-	if (instance->balanced_mode &&
+	if ((instance->perf_mode == MR_BALANCED_PERF_MODE) &&
 		atomic_read(&scp->device->device_busy) >
 		(io_info.data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))
 		cmd->request_desc->SCSIIO.MSIxIndex =
@@ -3164,7 +3172,7 @@ megasas_build_syspd_fusion(struct megasas_instance *instance,
 
 	cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
 
-	if (instance->balanced_mode &&
+	if ((instance->perf_mode == MR_BALANCED_PERF_MODE) &&
 		atomic_read(&scmd->device->device_busy) > MR_DEVICE_HIGH_IOPS_DEPTH)
 		cmd->request_desc->SCSIIO.MSIxIndex =
 			mega_mod64((atomic64_add_return(1, &instance->high_iops_outstanding) /